diff options
123 files changed, 2599 insertions, 3706 deletions
diff --git a/build.gradle b/build.gradle index 9185f1473d3..a7e1f05d425 100644 --- a/build.gradle +++ b/build.gradle @@ -342,14 +342,12 @@ subprojects { entry 'jetty-server' entry 'jetty-servlet' } - dependency('org.elasticsearch.client:transport:6.8.4') { - exclude 'org.elasticsearch.plugin:lang-mustache-client' + dependency('org.elasticsearch.client:elasticsearch-rest-high-level-client:7.9.3') { exclude 'commons-logging:commons-logging' - exclude 'org.elasticsearch.plugin:reindex-client' - exclude 'org.elasticsearch.plugin:rank-eval-client' } + dependency 'org.elasticsearch.plugin:transport-netty4-client:7.9.3' dependency 'org.elasticsearch:mocksocket:1.0' - dependency 'org.codelibs.elasticsearch.module:analysis-common:6.8.4' + dependency 'org.codelibs.elasticsearch.module:analysis-common:7.9.3' dependency 'org.eclipse.jgit:org.eclipse.jgit:5.9.0.202009080501-r' dependency 'org.tmatesoft.svnkit:svnkit:1.10.1' dependency 'org.hamcrest:hamcrest-all:1.3' diff --git a/server/sonar-main/build.gradle b/server/sonar-main/build.gradle index cb67a49fe04..2e11dd66e75 100644 --- a/server/sonar-main/build.gradle +++ b/server/sonar-main/build.gradle @@ -15,13 +15,11 @@ dependencies { compile 'com.hazelcast:hazelcast' compile 'commons-io:commons-io' compile 'commons-lang:commons-lang' - compile 'io.netty:netty-common' compile 'org.apache.logging.log4j:log4j-to-slf4j' compile 'org.apache.logging.log4j:log4j-api' - compile 'org.elasticsearch.client:transport' + compile 'org.elasticsearch.client:elasticsearch-rest-high-level-client' compile 'org.elasticsearch:elasticsearch' compile 'org.elasticsearch:elasticsearch-core' - compile 'org.elasticsearch.plugin:transport-netty4-client' compile 'org.slf4j:slf4j-api' compile 'org.yaml:snakeyaml' @@ -36,5 +34,7 @@ dependencies { testCompile 'org.assertj:assertj-core' testCompile 'org.awaitility:awaitility' testCompile 'org.mockito:mockito-core' + testCompile 'com.squareup.okhttp3:mockwebserver' + testCompile 'commons-logging:commons-logging:1.1.1' testCompile project(':sonar-testing-harness') } diff --git a/server/sonar-main/src/main/java/org/sonar/application/AppStateFactory.java b/server/sonar-main/src/main/java/org/sonar/application/AppStateFactory.java index 6a70a5ce795..b486faadbd8 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/AppStateFactory.java +++ b/server/sonar-main/src/main/java/org/sonar/application/AppStateFactory.java @@ -19,6 +19,7 @@ */ package org.sonar.application; +import com.google.common.collect.ImmutableSet; import com.google.common.net.HostAndPort; import java.util.Arrays; import java.util.Set; @@ -31,16 +32,18 @@ import org.sonar.application.es.EsConnector; import org.sonar.application.es.EsConnectorImpl; import org.sonar.process.ProcessId; import org.sonar.process.Props; +import org.sonar.process.cluster.NodeType; import org.sonar.process.cluster.hz.HazelcastMember; import org.sonar.process.cluster.hz.HazelcastMemberBuilder; import static java.util.Arrays.asList; import static org.sonar.process.ProcessProperties.Property.CLUSTER_HZ_HOSTS; -import static org.sonar.process.ProcessProperties.Property.CLUSTER_NAME; import static org.sonar.process.ProcessProperties.Property.CLUSTER_NODE_HOST; import static org.sonar.process.ProcessProperties.Property.CLUSTER_NODE_HZ_PORT; import static org.sonar.process.ProcessProperties.Property.CLUSTER_NODE_NAME; import static org.sonar.process.ProcessProperties.Property.CLUSTER_SEARCH_HOSTS; +import static org.sonar.process.ProcessProperties.Property.SEARCH_HOST; +import static org.sonar.process.ProcessProperties.Property.SEARCH_PORT; public class AppStateFactory { private final AppSettings settings; @@ -70,10 +73,17 @@ public class AppStateFactory { } private static EsConnector createEsConnector(Props props) { + NodeType nodeType = ClusterSettings.toNodeType(props); + if (nodeType == NodeType.SEARCH) { + String host = props.nonNullValue(SEARCH_HOST.getKey()); + String port = props.nonNullValue(SEARCH_PORT.getKey()); + return new EsConnectorImpl(ImmutableSet.of(HostAndPort.fromParts(host, Integer.valueOf(port)))); + } + String searchHosts = props.nonNullValue(CLUSTER_SEARCH_HOSTS.getKey()); Set<HostAndPort> hostAndPorts = Arrays.stream(searchHosts.split(",")) .map(HostAndPort::fromString) .collect(Collectors.toSet()); - return new EsConnectorImpl(props.nonNullValue(CLUSTER_NAME.getKey()), hostAndPorts); + return new EsConnectorImpl(hostAndPorts); } } diff --git a/server/sonar-main/src/main/java/org/sonar/application/ProcessLauncherImpl.java b/server/sonar-main/src/main/java/org/sonar/application/ProcessLauncherImpl.java index e0c4aab97f2..d6a23e5fe2f 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/ProcessLauncherImpl.java +++ b/server/sonar-main/src/main/java/org/sonar/application/ProcessLauncherImpl.java @@ -97,7 +97,7 @@ public class ProcessLauncherImpl implements ProcessLauncher { try { if (processId == ProcessId.ELASTICSEARCH) { checkArgument(esInstallation != null, "Incorrect configuration EsInstallation is null"); - EsConnectorImpl esConnector = new EsConnectorImpl(esInstallation.getClusterName(), singleton(HostAndPort.fromParts(esInstallation.getHost(), esInstallation.getPort()))); + EsConnectorImpl esConnector = new EsConnectorImpl(singleton(HostAndPort.fromParts(esInstallation.getHost(), esInstallation.getHttpPort()))); return new EsManagedProcess(process, processId, esConnector); } else { ProcessCommands commands = allProcessesCommands.createAfterClean(processId.getIpcIndex()); diff --git a/server/sonar-main/src/main/java/org/sonar/application/cluster/ClusterAppStateImpl.java b/server/sonar-main/src/main/java/org/sonar/application/cluster/ClusterAppStateImpl.java index 5efe6f0b999..6eec7ff5746 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/cluster/ClusterAppStateImpl.java +++ b/server/sonar-main/src/main/java/org/sonar/application/cluster/ClusterAppStateImpl.java @@ -200,8 +200,9 @@ public class ClusterAppStateImpl implements ClusterAppState { } private boolean isElasticSearchAvailable() { - ClusterHealthStatus clusterHealthStatus = esConnector.getClusterHealthStatus(); - return clusterHealthStatus.equals(ClusterHealthStatus.GREEN) || clusterHealthStatus.equals(ClusterHealthStatus.YELLOW); + return esConnector.getClusterHealthStatus() + .filter(t -> ClusterHealthStatus.GREEN.equals(t) || ClusterHealthStatus.YELLOW.equals(t)) + .isPresent(); } private class OperationalProcessListener implements EntryListener<ClusterProcess, Boolean> { diff --git a/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactoryImpl.java b/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactoryImpl.java index 37cd0c6c31e..353780521d3 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactoryImpl.java +++ b/server/sonar-main/src/main/java/org/sonar/application/command/CommandFactoryImpl.java @@ -143,9 +143,8 @@ public class CommandFactoryImpl implements CommandFactory { .addFromMandatoryProperty(props, SEARCH_JAVA_OPTS.getKey()) .addFromMandatoryProperty(props, SEARCH_JAVA_ADDITIONAL_OPTS.getKey())) .setEsYmlSettings(new EsYmlSettings(settingsMap)) - .setClusterName(settingsMap.get("cluster.name")) - .setHost(settingsMap.get("network.host")) - .setPort(Integer.parseInt(settingsMap.get("transport.port"))); + .setHost(settingsMap.get("http.host")) + .setHttpPort(Integer.parseInt(settingsMap.get("http.port"))); return esInstallation; } diff --git a/server/sonar-main/src/main/java/org/sonar/application/config/ClusterSettings.java b/server/sonar-main/src/main/java/org/sonar/application/config/ClusterSettings.java index 85d7078381c..997e68a5f50 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/config/ClusterSettings.java +++ b/server/sonar-main/src/main/java/org/sonar/application/config/ClusterSettings.java @@ -138,7 +138,7 @@ public class ClusterSettings implements Consumer<Props> { return addressAndPort; } - private static NodeType toNodeType(Props props) { + public static NodeType toNodeType(Props props) { String nodeTypeValue = requireValue(props, CLUSTER_NODE_TYPE); if (!NodeType.isValid(nodeTypeValue)) { throw new MessageException(format("Invalid value for property %s: [%s], only [%s] are allowed", CLUSTER_NODE_TYPE.getKey(), nodeTypeValue, diff --git a/server/sonar-main/src/main/java/org/sonar/application/es/EsConnector.java b/server/sonar-main/src/main/java/org/sonar/application/es/EsConnector.java index 6e289407904..78544b302a5 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/es/EsConnector.java +++ b/server/sonar-main/src/main/java/org/sonar/application/es/EsConnector.java @@ -19,9 +19,11 @@ */ package org.sonar.application.es; +import java.util.Optional; import org.elasticsearch.cluster.health.ClusterHealthStatus; public interface EsConnector { - ClusterHealthStatus getClusterHealthStatus(); + Optional<ClusterHealthStatus> getClusterHealthStatus(); + void stop(); } diff --git a/server/sonar-main/src/main/java/org/sonar/application/es/EsConnectorImpl.java b/server/sonar-main/src/main/java/org/sonar/application/es/EsConnectorImpl.java index 8252014f12d..bacc25c85b6 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/es/EsConnectorImpl.java +++ b/server/sonar-main/src/main/java/org/sonar/application/es/EsConnectorImpl.java @@ -20,121 +20,83 @@ package org.sonar.application.es; import com.google.common.net.HostAndPort; -import io.netty.util.ThreadDeathWatcher; -import io.netty.util.concurrent.GlobalEventExecutor; -import java.net.InetAddress; -import java.net.UnknownHostException; +import java.io.IOException; +import java.util.Arrays; +import java.util.Optional; import java.util.Set; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import org.apache.http.HttpHost; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; -import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.common.network.NetworkModule; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.transport.Netty4Plugin; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static java.lang.String.format; -import static java.util.Collections.singletonList; -import static java.util.Collections.unmodifiableList; import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; public class EsConnectorImpl implements EsConnector { private static final Logger LOG = LoggerFactory.getLogger(EsConnectorImpl.class); - private final AtomicReference<TransportClient> transportClient = new AtomicReference<>(null); - private final String clusterName; + private final AtomicReference<RestHighLevelClient> restClient = new AtomicReference<>(null); private final Set<HostAndPort> hostAndPorts; - public EsConnectorImpl(String clusterName, Set<HostAndPort> hostAndPorts) { - this.clusterName = clusterName; + public EsConnectorImpl(Set<HostAndPort> hostAndPorts) { this.hostAndPorts = hostAndPorts; } @Override - public ClusterHealthStatus getClusterHealthStatus() { - return getTransportClient().admin().cluster() - .health(new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.YELLOW).timeout(timeValueSeconds(30))) - .actionGet().getStatus(); + public Optional<ClusterHealthStatus> getClusterHealthStatus() { + try { + ClusterHealthResponse healthResponse = getRestHighLevelClient().cluster() + .health(new ClusterHealthRequest().waitForYellowStatus().timeout(timeValueSeconds(30)), RequestOptions.DEFAULT); + return Optional.of(healthResponse.getStatus()); + } catch (IOException e) { + LOG.trace("Failed to check health status ", e); + return Optional.empty(); + } } @Override public void stop() { - transportClient.set(null); - } - - private TransportClient getTransportClient() { - TransportClient res = this.transportClient.get(); - if (res == null) { - res = buildTransportClient(); - if (this.transportClient.compareAndSet(null, res)) { - return res; + RestHighLevelClient restHighLevelClient = restClient.get(); + if (restHighLevelClient != null) { + try { + restHighLevelClient.close(); + } catch (IOException e) { + LOG.warn("Error occurred while closing Rest Client", e); } - return this.transportClient.get(); } - return res; } - private TransportClient buildTransportClient() { - Settings.Builder esSettings = Settings.builder(); - - // mandatory property defined by bootstrap process - esSettings.put("cluster.name", clusterName); - - TransportClient nativeClient = new MinimalTransportClient(esSettings.build(), hostAndPorts); - if (LOG.isDebugEnabled()) { - LOG.debug("Connected to Elasticsearch node: [{}]", displayedAddresses(nativeClient)); + private RestHighLevelClient getRestHighLevelClient() { + RestHighLevelClient res = this.restClient.get(); + if (res != null) { + return res; } - return nativeClient; - } - private static String displayedAddresses(TransportClient nativeClient) { - return nativeClient.transportAddresses().stream().map(TransportAddress::toString).collect(Collectors.joining(", ")); + RestHighLevelClient restHighLevelClient = buildRestHighLevelClient(); + this.restClient.set(restHighLevelClient); + return restHighLevelClient; } - private static class MinimalTransportClient extends TransportClient { - - public MinimalTransportClient(Settings settings, Set<HostAndPort> hostAndPorts) { - super(settings, unmodifiableList(singletonList(Netty4Plugin.class))); + private RestHighLevelClient buildRestHighLevelClient() { + HttpHost[] httpHosts = hostAndPorts.stream() + .map(hostAndPort -> new HttpHost(hostAndPort.getHost(), hostAndPort.getPortOrDefault(9001))) + .toArray(HttpHost[]::new); - boolean connectedToOneHost = false; - for (HostAndPort hostAndPort : hostAndPorts) { - try { - addTransportAddress(new TransportAddress(InetAddress.getByName(hostAndPort.getHost()), hostAndPort.getPortOrDefault(9001))); - connectedToOneHost = true; - } catch (UnknownHostException e) { - LOG.debug("Can not resolve host [" + hostAndPort.getHost() + "]", e); - } - } - if (!connectedToOneHost) { - throw new IllegalStateException(format("Can not connect to one node from [%s]", - hostAndPorts.stream() - .map(h -> format("%s:%d", h.getHost(), h.getPortOrDefault(9001))) - .collect(Collectors.joining(",")))); - } - } - - @Override - public void close() { - super.close(); - if (!NetworkModule.TRANSPORT_TYPE_SETTING.exists(settings) - || NetworkModule.TRANSPORT_TYPE_SETTING.get(settings).equals(Netty4Plugin.NETTY_TRANSPORT_NAME)) { - try { - GlobalEventExecutor.INSTANCE.awaitInactivity(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - try { - ThreadDeathWatcher.awaitInactivity(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } + if (LOG.isDebugEnabled()) { + String addresses = Arrays.stream(httpHosts) + .map(t -> t.getHostName() + ":" + t.getPort()) + .collect(Collectors.joining(", ")); + LOG.debug("Connected to Elasticsearch node: [{}]", addresses); } + return new RestHighLevelClient(RestClient.builder(httpHosts)); } + } diff --git a/server/sonar-main/src/main/java/org/sonar/application/es/EsInstallation.java b/server/sonar-main/src/main/java/org/sonar/application/es/EsInstallation.java index 6d084d23cba..5657f094adb 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/es/EsInstallation.java +++ b/server/sonar-main/src/main/java/org/sonar/application/es/EsInstallation.java @@ -52,7 +52,7 @@ public class EsInstallation { private Properties log4j2Properties; private String clusterName; private String host; - private int port; + private int httpPort; public EsInstallation(Props props) { File sqHomeDir = props.nonNullValueAsFile(PATH_HOME.getKey()); @@ -152,15 +152,6 @@ public class EsInstallation { return this; } - public String getClusterName() { - return clusterName; - } - - public EsInstallation setClusterName(String clusterName) { - this.clusterName = clusterName; - return this; - } - public String getHost() { return host; } @@ -170,12 +161,12 @@ public class EsInstallation { return this; } - public int getPort() { - return port; + public int getHttpPort() { + return httpPort; } - public EsInstallation setPort(int port) { - this.port = port; + public EsInstallation setHttpPort(int httpPort) { + this.httpPort = httpPort; return this; } } diff --git a/server/sonar-main/src/main/java/org/sonar/application/es/EsSettings.java b/server/sonar-main/src/main/java/org/sonar/application/es/EsSettings.java index a852fdf4efc..1a7b658c70a 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/es/EsSettings.java +++ b/server/sonar-main/src/main/java/org/sonar/application/es/EsSettings.java @@ -36,9 +36,9 @@ import static org.sonar.process.ProcessProperties.Property.CLUSTER_NAME; import static org.sonar.process.ProcessProperties.Property.CLUSTER_NODE_NAME; import static org.sonar.process.ProcessProperties.Property.CLUSTER_SEARCH_HOSTS; import static org.sonar.process.ProcessProperties.Property.SEARCH_HOST; -import static org.sonar.process.ProcessProperties.Property.SEARCH_HTTP_PORT; import static org.sonar.process.ProcessProperties.Property.SEARCH_INITIAL_STATE_TIMEOUT; import static org.sonar.process.ProcessProperties.Property.SEARCH_PORT; +import static org.sonar.process.ProcessProperties.Property.SEARCH_TRANSPORT_PORT; public class EsSettings { @@ -75,8 +75,8 @@ public class EsSettings { public Map<String, String> build() { Map<String, String> builder = new HashMap<>(); configureFileSystem(builder); - configureNetwork(builder); - configureCluster(builder); + InetAddress host = configureNetwork(builder); + configureCluster(builder, host); configureOthers(builder); return builder; } @@ -86,33 +86,27 @@ public class EsSettings { builder.put("path.logs", fileSystem.getLogDirectory().getAbsolutePath()); } - private void configureNetwork(Map<String, String> builder) { + private InetAddress configureNetwork(Map<String, String> builder) { InetAddress host = readHost(); - int port = Integer.parseInt(props.nonNullValue(SEARCH_PORT.getKey())); - LOGGER.info("Elasticsearch listening on {}:{}", host, port); + int httpPort = Integer.parseInt(props.nonNullValue(SEARCH_PORT.getKey())); + LOGGER.info("Elasticsearch listening on {}:{}", host, httpPort); - // FIXME no need to open TCP port unless running DCE - // TCP is used by main process to check ES is up => probably has to use HTTP now - builder.put("transport.port", valueOf(port)); - builder.put("transport.host", valueOf(host.getHostAddress())); - builder.put("network.host", valueOf(host.getHostAddress())); + // see https://github.com/lmenezes/elasticsearch-kopf/issues/195 + builder.put("http.cors.enabled", valueOf(true)); + builder.put("http.cors.allow-origin", "*"); + builder.put("http.host", host.getHostAddress()); + builder.put("http.port", valueOf(httpPort)); + builder.put("network.host", valueOf(host.getHostAddress())); // Elasticsearch sets the default value of TCP reuse address to true only on non-MSWindows machines, but why ? builder.put("network.tcp.reuse_address", valueOf(true)); - int httpPort = props.valueAsInt(SEARCH_HTTP_PORT.getKey(), -1); - if (httpPort < 0) { - // standard configuration - builder.put("http.enabled", valueOf(false)); - } else { - LOGGER.warn("Elasticsearch HTTP connector is enabled on port {}. MUST NOT BE USED FOR PRODUCTION", httpPort); - // see https://github.com/lmenezes/elasticsearch-kopf/issues/195 - builder.put("http.cors.enabled", valueOf(true)); - builder.put("http.cors.allow-origin", "*"); - builder.put("http.enabled", valueOf(true)); - builder.put("http.host", host.getHostAddress()); - builder.put("http.port", valueOf(httpPort)); - } + // FIXME remove definition of transport properties when Web and CE have moved to ES Rest client + int tcpPort = props.valueAsInt(SEARCH_TRANSPORT_PORT.getKey(), 9002); + builder.put("transport.port", valueOf(tcpPort)); + builder.put("transport.host", valueOf(host.getHostAddress())); + + return host; } private InetAddress readHost() { @@ -124,7 +118,7 @@ public class EsSettings { } } - private void configureCluster(Map<String, String> builder) { + private void configureCluster(Map<String, String> builder, InetAddress host) { // Default value in a standalone mode, not overridable String initialStateTimeOut = "30s"; @@ -132,6 +126,10 @@ public class EsSettings { if (clusterEnabled) { initialStateTimeOut = props.value(SEARCH_INITIAL_STATE_TIMEOUT.getKey(), "120s"); + int tcpPort = props.valueAsInt(SEARCH_TRANSPORT_PORT.getKey(), 9002); + builder.put("transport.port", valueOf(tcpPort)); + builder.put("transport.host", valueOf(host.getHostAddress())); + String hosts = props.value(CLUSTER_SEARCH_HOSTS.getKey(), ""); LOGGER.info("Elasticsearch cluster enabled. Connect to hosts [{}]", hosts); builder.put("discovery.seed_hosts", hosts); diff --git a/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java b/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java index 6fe8f1acc35..36008dc3988 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java +++ b/server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java @@ -20,8 +20,9 @@ package org.sonar.application.process; import java.util.concurrent.atomic.AtomicBoolean; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.discovery.MasterNotDiscoveredException; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.rest.RestStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.application.es.EsConnector; @@ -85,21 +86,14 @@ public class EsManagedProcess extends AbstractManagedProcess { private Status checkStatus() { try { - switch (esConnector.getClusterHealthStatus()) { - case GREEN: - return GREEN; - case YELLOW: - return YELLOW; - case RED: - return RED; - default: - return KO; - } - } catch (NoNodeAvailableException e) { - return CONNECTION_REFUSED; - } catch (MasterNotDiscoveredException e) { - if (firstMasterNotDiscoveredLog.getAndSet(false)) { - LOG.info("Elasticsearch is waiting for a master to be elected. Did you start all the search nodes ?"); + return esConnector.getClusterHealthStatus() + .map(EsManagedProcess::convert) + .orElse(CONNECTION_REFUSED); + } catch (ElasticsearchStatusException e) { + if (e.status() == RestStatus.SERVICE_UNAVAILABLE && e.getMessage().contains("type=master_not_discovered_exception")) { + if (firstMasterNotDiscoveredLog.getAndSet(false)) { + LOG.info("Elasticsearch is waiting for a master to be elected. Did you start all the search nodes ?"); + } } return KO; } catch (Exception e) { @@ -108,6 +102,19 @@ public class EsManagedProcess extends AbstractManagedProcess { } } + private static Status convert(ClusterHealthStatus clusterHealthStatus) { + switch (clusterHealthStatus) { + case GREEN: + return GREEN; + case YELLOW: + return YELLOW; + case RED: + return RED; + default: + return KO; + } + } + enum Status { CONNECTION_REFUSED, KO, RED, YELLOW, GREEN } diff --git a/server/sonar-main/src/test/java/org/sonar/application/AppStateFactoryTest.java b/server/sonar-main/src/test/java/org/sonar/application/AppStateFactoryTest.java index 1ae9c798b9a..48cbf0754e9 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/AppStateFactoryTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/AppStateFactoryTest.java @@ -51,7 +51,7 @@ public class AppStateFactoryTest { settings.set(CLUSTER_NODE_HOST.getKey(), ip.get().getHostAddress()); settings.set(CLUSTER_HZ_HOSTS.getKey(), ip.get().getHostAddress()); settings.set(CLUSTER_NAME.getKey(), "foo"); - settings.set(CLUSTER_SEARCH_HOSTS.getKey(), "localhost:9001"); + settings.set(CLUSTER_SEARCH_HOSTS.getKey(), "localhost:9002"); AppState appState = underTest.create(); assertThat(appState).isInstanceOf(ClusterAppStateImpl.class); diff --git a/server/sonar-main/src/test/java/org/sonar/application/ProcessLauncherImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/ProcessLauncherImplTest.java index 30360924a9a..9612aff58d5 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/ProcessLauncherImplTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/ProcessLauncherImplTest.java @@ -200,8 +200,7 @@ public class ProcessLauncherImplTest { .setEsJvmOptions(mock(EsJvmOptions.class)) .setLog4j2Properties(new Properties()) .setHost("localhost") - .setPort(9001) - .setClusterName("sonarqube")); + .setHttpPort(9001)); return command; } @@ -212,8 +211,7 @@ public class ProcessLauncherImplTest { .set("sonar.path.data", this.temp.newFolder("data").getAbsolutePath()) .set("sonar.path.temp", tempFolder.getAbsolutePath()) .set("sonar.path.logs", this.temp.newFolder("logs").getAbsolutePath())) - .setClusterName("cluster") - .setPort(9001) + .setHttpPort(9001) .setHost("localhost") .setEsYmlSettings(new EsYmlSettings(new HashMap<>())) .setEsJvmOptions(new EsJvmOptions(new Props(new Properties()), tempFolder)) diff --git a/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java index ee0537edc3a..bbe96d163b0 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java @@ -131,9 +131,8 @@ public class CommandFactoryImplTest { EsScriptCommand esCommand = (EsScriptCommand) command; EsInstallation esConfig = esCommand.getEsInstallation(); - assertThat(esConfig.getClusterName()).isEqualTo("sonarqube"); assertThat(esConfig.getHost()).isNotEmpty(); - assertThat(esConfig.getPort()).isEqualTo(9001); + assertThat(esConfig.getHttpPort()).isEqualTo(9001); assertThat(esConfig.getEsJvmOptions().getAll()) // enforced values .contains("-XX:+UseConcMarkSweepGC", "-Dfile.encoding=UTF-8") @@ -167,9 +166,8 @@ public class CommandFactoryImplTest { JavaCommand<?> esCommand = (JavaCommand<?>) command; EsInstallation esConfig = esCommand.getEsInstallation(); - assertThat(esConfig.getClusterName()).isEqualTo("sonarqube"); assertThat(esConfig.getHost()).isNotEmpty(); - assertThat(esConfig.getPort()).isEqualTo(9001); + assertThat(esConfig.getHttpPort()).isEqualTo(9001); assertThat(esConfig.getEsJvmOptions().getAll()) // enforced values .contains("-XX:+UseConcMarkSweepGC", "-Dfile.encoding=UTF-8") @@ -209,8 +207,7 @@ public class CommandFactoryImplTest { AbstractCommand esCommand = newFactory(props).createEsCommand(); EsInstallation esConfig = esCommand.getEsInstallation(); - assertThat(esConfig.getClusterName()).isEqualTo("foo"); - assertThat(esConfig.getPort()).isEqualTo(1234); + assertThat(esConfig.getHttpPort()).isEqualTo(1234); assertThat(esConfig.getEsJvmOptions().getAll()) // enforced values .contains("-XX:+UseConcMarkSweepGC", "-Dfile.encoding=UTF-8") diff --git a/server/sonar-main/src/test/java/org/sonar/application/es/EsConnectorImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/es/EsConnectorImplTest.java new file mode 100644 index 00000000000..3bacc724278 --- /dev/null +++ b/server/sonar-main/src/test/java/org/sonar/application/es/EsConnectorImplTest.java @@ -0,0 +1,92 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.application.es; + +import com.google.common.collect.Sets; +import com.google.common.net.HostAndPort; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.junit.After; +import org.junit.Rule; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class EsConnectorImplTest { + + private static final String JSON_SUCCESS_RESPONSE = "{" + + " \"cluster_name\" : \"testcluster\"," + + " \"status\" : \"yellow\"," + + " \"timed_out\" : false," + + " \"number_of_nodes\" : 1," + + " \"number_of_data_nodes\" : 1," + + " \"active_primary_shards\" : 1," + + " \"active_shards\" : 1," + + " \"relocating_shards\" : 0," + + " \"initializing_shards\" : 0," + + " \"unassigned_shards\" : 1," + + " \"delayed_unassigned_shards\": 0," + + " \"number_of_pending_tasks\" : 0," + + " \"number_of_in_flight_fetch\": 0," + + " \"task_max_waiting_in_queue_millis\": 0," + + " \"active_shards_percent_as_number\": 50.0" + + "}"; + + private static final String JSON_ERROR_RESPONSE = "{" + + " \"error\" : \"i-have-a-bad-feelings-about-this\"" + + "}"; + + @Rule + public MockWebServer mockWebServer = new MockWebServer(); + + EsConnectorImpl underTest = new EsConnectorImpl(Sets.newHashSet(HostAndPort.fromParts(mockWebServer.getHostName(), mockWebServer.getPort()))); + + @After + public void after() { + underTest.stop(); + } + + @Test + public void should_rethrow_if_es_exception() { + mockServerResponse(500, JSON_ERROR_RESPONSE); + + assertThatThrownBy(() -> underTest.getClusterHealthStatus()) + .isInstanceOf(ElasticsearchStatusException.class); + } + + @Test + public void should_return_status() { + mockServerResponse(200, JSON_SUCCESS_RESPONSE); + + assertThat(underTest.getClusterHealthStatus()) + .hasValue(ClusterHealthStatus.YELLOW); + } + + private void mockServerResponse(int httpCode, String jsonResponse) { + mockWebServer.enqueue(new MockResponse() + .setResponseCode(httpCode) + .setBody(jsonResponse) + .setHeader("Content-Type", "application/json")); + } + +} diff --git a/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java b/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java index cfc7800d52e..306f6a79332 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java @@ -56,7 +56,6 @@ import static org.sonar.process.ProcessProperties.Property.PATH_HOME; import static org.sonar.process.ProcessProperties.Property.PATH_LOGS; import static org.sonar.process.ProcessProperties.Property.PATH_TEMP; import static org.sonar.process.ProcessProperties.Property.SEARCH_HOST; -import static org.sonar.process.ProcessProperties.Property.SEARCH_HTTP_PORT; import static org.sonar.process.ProcessProperties.Property.SEARCH_INITIAL_STATE_TIMEOUT; import static org.sonar.process.ProcessProperties.Property.SEARCH_PORT; @@ -138,9 +137,14 @@ public class EsSettingsTest { EsSettings esSettings = new EsSettings(props, new EsInstallation(props), system); Map<String, String> generated = esSettings.build(); - assertThat(generated.get("transport.port")).isEqualTo("1234"); + + // FIXME transport.port and transport.host should not be set in standalone + assertThat(generated.get("transport.port")).isEqualTo("9002"); assertThat(generated.get("transport.host")).isEqualTo("127.0.0.1"); + assertThat(generated.get("http.port")).isEqualTo("1234"); + assertThat(generated.get("http.host")).isEqualTo("127.0.0.1"); + // no cluster, but cluster and node names are set though assertThat(generated.get("cluster.name")).isEqualTo("sonarqube"); assertThat(generated.get("node.name")).isEqualTo("sonarqube"); @@ -150,9 +154,6 @@ public class EsSettingsTest { assertThat(generated.get("path.home")).isNull(); assertThat(generated.get("path.conf")).isNull(); - // http is disabled for security reasons - assertThat(generated.get("http.enabled")).isEqualTo("false"); - assertThat(generated.get("discovery.seed_hosts")).isNull(); assertThat(generated.get("discovery.initial_state_timeout")).isEqualTo("30s"); @@ -274,39 +275,51 @@ public class EsSettingsTest { } @Test - public void enable_http_connector() throws Exception { - Props props = minProps(CLUSTER_DISABLED); - props.set(SEARCH_HTTP_PORT.getKey(), "9010"); + @UseDataProvider("clusterEnabledOrNot") + public void enable_http_connector_on_port_9001_by_default(boolean clusterEnabled) throws Exception { + Props props = minProps(clusterEnabled); Map<String, String> settings = new EsSettings(props, new EsInstallation(props), system).build(); - assertThat(settings.get("http.port")).isEqualTo("9010"); + assertThat(settings.get("http.port")).isEqualTo("9001"); assertThat(settings.get("http.host")).isEqualTo("127.0.0.1"); - assertThat(settings.get("http.enabled")).isEqualTo("true"); } @Test - public void enable_http_connector_different_host() throws Exception { - Props props = minProps(CLUSTER_DISABLED); - props.set(SEARCH_HTTP_PORT.getKey(), "9010"); + @UseDataProvider("clusterEnabledOrNot") + public void enable_http_connector_on_specified_port(boolean clusterEnabled) throws Exception { + String port = "" + new Random().nextInt(49151); + Props props = minProps(clusterEnabled); + props.set(SEARCH_PORT.getKey(), port); + Map<String, String> settings = new EsSettings(props, new EsInstallation(props), System2.INSTANCE).build(); + + assertThat(settings.get("http.port")).isEqualTo(port); + assertThat(settings.get("http.host")).isEqualTo("127.0.0.1"); + } + + @Test + @UseDataProvider("clusterEnabledOrNot") + public void enable_http_connector_different_host(boolean clusterEnabled) throws Exception { + Props props = minProps(clusterEnabled); props.set(SEARCH_HOST.getKey(), "127.0.0.2"); Map<String, String> settings = new EsSettings(props, new EsInstallation(props), system).build(); - assertThat(settings.get("http.port")).isEqualTo("9010"); + assertThat(settings.get("http.port")).isEqualTo("9001"); assertThat(settings.get("http.host")).isEqualTo("127.0.0.2"); - assertThat(settings.get("http.enabled")).isEqualTo("true"); } @Test - public void enable_seccomp_filter_by_default() throws Exception { - Props props = minProps(CLUSTER_DISABLED); + @UseDataProvider("clusterEnabledOrNot") + public void enable_seccomp_filter_by_default(boolean clusterEnabled) throws Exception { + Props props = minProps(clusterEnabled); Map<String, String> settings = new EsSettings(props, new EsInstallation(props), system).build(); assertThat(settings.get("bootstrap.system_call_filter")).isNull(); } @Test - public void disable_seccomp_filter_if_configured_in_search_additional_props() throws Exception { - Props props = minProps(CLUSTER_DISABLED); + @UseDataProvider("clusterEnabledOrNot") + public void disable_seccomp_filter_if_configured_in_search_additional_props(boolean clusterEnabled) throws Exception { + Props props = minProps(clusterEnabled); props.set("sonar.search.javaAdditionalOpts", "-Xmx1G -Dbootstrap.system_call_filter=false -Dfoo=bar"); Map<String, String> settings = new EsSettings(props, new EsInstallation(props), system).build(); @@ -314,8 +327,9 @@ public class EsSettingsTest { } @Test - public void disable_mmap_if_configured_in_search_additional_props() throws Exception { - Props props = minProps(CLUSTER_DISABLED); + @UseDataProvider("clusterEnabledOrNot") + public void disable_mmap_if_configured_in_search_additional_props(boolean clusterEnabled) throws Exception { + Props props = minProps(clusterEnabled); props.set("sonar.search.javaAdditionalOpts", "-Dnode.store.allow_mmap=false"); Map<String, String> settings = new EsSettings(props, new EsInstallation(props), system).build(); diff --git a/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java b/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java index 967b319e8b6..d4654af2477 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java @@ -25,9 +25,10 @@ import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.AppenderBase; import java.util.ArrayList; import java.util.List; -import org.elasticsearch.client.transport.NoNodeAvailableException; +import java.util.Optional; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.discovery.MasterNotDiscoveredException; +import org.elasticsearch.rest.RestStatus; import org.junit.Test; import org.slf4j.LoggerFactory; import org.sonar.application.es.EsConnector; @@ -41,9 +42,17 @@ import static org.mockito.Mockito.when; public class EsManagedProcessTest { @Test + public void isOperational_should_return_false_if_status_is_unknown() { + EsConnector esConnector = mock(EsConnector.class); + when(esConnector.getClusterHealthStatus()).thenReturn(Optional.empty()); + EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector); + assertThat(underTest.isOperational()).isFalse(); + } + + @Test public void isOperational_should_return_false_if_Elasticsearch_is_RED() { EsConnector esConnector = mock(EsConnector.class); - when(esConnector.getClusterHealthStatus()).thenReturn(ClusterHealthStatus.RED); + when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.RED)); EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector); assertThat(underTest.isOperational()).isFalse(); } @@ -51,7 +60,7 @@ public class EsManagedProcessTest { @Test public void isOperational_should_return_true_if_Elasticsearch_is_YELLOW() { EsConnector esConnector = mock(EsConnector.class); - when(esConnector.getClusterHealthStatus()).thenReturn(ClusterHealthStatus.YELLOW); + when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.YELLOW)); EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector); assertThat(underTest.isOperational()).isTrue(); } @@ -59,7 +68,7 @@ public class EsManagedProcessTest { @Test public void isOperational_should_return_true_if_Elasticsearch_is_GREEN() { EsConnector esConnector = mock(EsConnector.class); - when(esConnector.getClusterHealthStatus()).thenReturn(ClusterHealthStatus.GREEN); + when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.GREEN)); EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector); assertThat(underTest.isOperational()).isTrue(); } @@ -67,11 +76,11 @@ public class EsManagedProcessTest { @Test public void isOperational_should_return_true_if_Elasticsearch_was_GREEN_once() { EsConnector esConnector = mock(EsConnector.class); - when(esConnector.getClusterHealthStatus()).thenReturn(ClusterHealthStatus.GREEN); + when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.GREEN)); EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector); assertThat(underTest.isOperational()).isTrue(); - when(esConnector.getClusterHealthStatus()).thenReturn(ClusterHealthStatus.RED); + when(esConnector.getClusterHealthStatus()).thenReturn(Optional.of(ClusterHealthStatus.RED)); assertThat(underTest.isOperational()).isTrue(); } @@ -79,8 +88,8 @@ public class EsManagedProcessTest { public void isOperational_should_retry_if_Elasticsearch_is_unreachable() { EsConnector esConnector = mock(EsConnector.class); when(esConnector.getClusterHealthStatus()) - .thenThrow(new NoNodeAvailableException("test")) - .thenReturn(ClusterHealthStatus.GREEN); + .thenReturn(Optional.empty()) + .thenReturn(Optional.of(ClusterHealthStatus.GREEN)); EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector); assertThat(underTest.isOperational()).isTrue(); } @@ -105,7 +114,7 @@ public class EsManagedProcessTest { EsConnector esConnector = mock(EsConnector.class); when(esConnector.getClusterHealthStatus()) - .thenThrow(new MasterNotDiscoveredException("Master not elected -test-")); + .thenThrow(new ElasticsearchStatusException("foobar[type=master_not_discovered_exception,acme]...", RestStatus.SERVICE_UNAVAILABLE)); EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector); assertThat(underTest.isOperational()).isFalse(); diff --git a/server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java b/server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java index 0c25ed6b843..3c6583b2050 100644 --- a/server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java +++ b/server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java @@ -74,7 +74,8 @@ public class ProcessProperties { SEARCH_HOST("sonar.search.host", InetAddress.getLoopbackAddress().getHostAddress()), SEARCH_PORT("sonar.search.port", "9001"), - SEARCH_HTTP_PORT("sonar.search.httpPort"), + // FIXME default is 0 until we move out of usage of TransportClient and we can put the expected default: 9002 + SEARCH_TRANSPORT_PORT("sonar.search.transportPort", "0"), SEARCH_JAVA_OPTS("sonar.search.javaOpts", "-Xmx512m -Xms512m -XX:MaxDirectMemorySize=256m -XX:+HeapDumpOnOutOfMemoryError"), SEARCH_JAVA_ADDITIONAL_OPTS("sonar.search.javaAdditionalOpts", ""), SEARCH_REPLICAS("sonar.search.replicas"), @@ -224,6 +225,8 @@ public class ProcessProperties { } fixPortIfZero(props, Property.SEARCH_HOST.getKey(), Property.SEARCH_PORT.getKey()); + // FIXME remove when transport is not used anymore in non-DCE editions: sonar.search.transportPort must not support port 0 + fixPortIfZero(props, Property.SEARCH_HOST.getKey(), Property.SEARCH_TRANSPORT_PORT.getKey()); } private Properties defaults() { diff --git a/server/sonar-server-common/build.gradle b/server/sonar-server-common/build.gradle index 5e027f81878..a6ffdd9d140 100644 --- a/server/sonar-server-common/build.gradle +++ b/server/sonar-server-common/build.gradle @@ -15,7 +15,7 @@ dependencies { compile 'com.google.guava:guava' compile 'org.slf4j:slf4j-api' compile 'com.squareup.okhttp3:okhttp' - compile 'org.elasticsearch.client:transport' + compile 'org.elasticsearch.client:elasticsearch-rest-high-level-client' compile project(':server:sonar-db-dao') compile project(':server:sonar-db-migration') compile project(':server:sonar-process') @@ -26,6 +26,7 @@ dependencies { compileOnly 'com.google.code.findbugs:jsr305' + testCompile 'org.elasticsearch.plugin:transport-netty4-client' testCompile 'ch.qos.logback:logback-core' testCompile 'com.google.code.findbugs:jsr305' testCompile 'com.squareup.okhttp3:mockwebserver' @@ -46,6 +47,7 @@ dependencies { testFixturesApi testFixtures(project(':server:sonar-db-dao')) testFixturesCompileOnly 'com.google.code.findbugs:jsr305' + testFixturesImplementation 'org.elasticsearch.plugin:transport-netty4-client' testFixturesImplementation 'org.codelibs.elasticsearch.module:analysis-common' testFixturesImplementation 'org.elasticsearch:mocksocket' diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/component/index/ComponentIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/component/index/ComponentIndexer.java index 8e01790e79e..a24c8e335bd 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/component/index/ComponentIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/component/index/ComponentIndexer.java @@ -27,8 +27,9 @@ import java.util.HashSet; import java.util.List; import java.util.Set; import javax.annotation.Nullable; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.sonar.core.util.stream.MoreCollectors; import org.sonar.db.DbClient; import org.sonar.db.DbSession; @@ -152,9 +153,9 @@ public class ComponentIndexer implements ProjectIndexer, NeedAuthorizationIndexe } private void addProjectDeletionToBulkIndexer(BulkIndexer bulkIndexer, String projectUuid) { - SearchRequestBuilder searchRequest = esClient.prepareSearch(TYPE_COMPONENT.getMainType()) - .setQuery(QueryBuilders.termQuery(ComponentIndexDefinition.FIELD_PROJECT_UUID, projectUuid)) - .setRouting(AuthorizationDoc.idOf(projectUuid)); + SearchRequest searchRequest = EsClient.prepareSearch(TYPE_COMPONENT.getMainType()) + .source(new SearchSourceBuilder().query(QueryBuilders.termQuery(ComponentIndexDefinition.FIELD_PROJECT_UUID, projectUuid))) + .routing(AuthorizationDoc.idOf(projectUuid)); bulkIndexer.addDeletion(searchRequest); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/BulkIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/BulkIndexer.java index 115887d2a7e..ba0d8d3b4cb 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/BulkIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/BulkIndexer.java @@ -31,8 +31,10 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.bulk.BackoffPolicy; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkProcessor; @@ -40,12 +42,13 @@ import org.elasticsearch.action.bulk.BulkProcessor.Listener; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.delete.DeleteRequestBuilder; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.ClearScrollRequest; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; @@ -74,7 +77,8 @@ public class BulkIndexer { private static final String REFRESH_INTERVAL_SETTING = "index.refresh_interval"; private static final int DEFAULT_NUMBER_OF_SHARDS = 5; - private final EsClient client; + private final EsClient esClient; + private final IndexType indexType; private final BulkProcessor bulkProcessor; private final IndexingResult result = new IndexingResult(); @@ -86,12 +90,14 @@ public class BulkIndexer { } public BulkIndexer(EsClient client, IndexType indexType, Size size, IndexingListener indexingListener) { - this.client = client; + this.esClient = client; this.indexType = indexType; this.sizeHandler = size.createHandler(Runtime2.INSTANCE); this.indexingListener = indexingListener; BulkProcessorListener bulkProcessorListener = new BulkProcessorListener(); - this.bulkProcessor = BulkProcessor.builder(client.nativeClient(), bulkProcessorListener) + this.bulkProcessor = BulkProcessor.builder( + client::bulkAsync, + bulkProcessorListener) .setBackoffPolicy(BackoffPolicy.exponentialBackoff()) .setBulkSize(FLUSH_BYTE_SIZE) .setBulkActions(FLUSH_ACTIONS) @@ -118,7 +124,9 @@ public class BulkIndexer { Thread.currentThread().interrupt(); throw new IllegalStateException("Elasticsearch bulk requests still being executed after 1 minute", e); } - client.prepareRefresh(indexType.getMainType().getIndex()).get(); + + esClient.refresh(indexType.getMainType().getIndex()); + sizeHandler.afterStop(this); indexingListener.onFinish(result); return result; @@ -139,50 +147,53 @@ public class BulkIndexer { bulkProcessor.add(request); } - public void addDeletion(SearchRequestBuilder searchRequest) { + public void addDeletion(SearchRequest searchRequest) { // TODO to be replaced by delete_by_query that is back in ES5 searchRequest - .addSort("_doc", SortOrder.ASC) - .setScroll(TimeValue.timeValueMinutes(5)) - .setSize(100) + .scroll(TimeValue.timeValueMinutes(5)) + .source() + .sort("_doc", SortOrder.ASC) + .size(100) // load only doc ids, not _source fields - .setFetchSource(false); + .fetchSource(false); // this search is synchronous. An optimization would be to be non-blocking, // but it requires to tracking pending requests in close(). // Same semaphore can't be reused because of potential deadlock (requires to acquire // two locks) - SearchResponse searchResponse = searchRequest.get(); + SearchResponse searchResponse = esClient.search(searchRequest); while (true) { SearchHit[] hits = searchResponse.getHits().getHits(); for (SearchHit hit : hits) { DocumentField routing = hit.field("_routing"); - DeleteRequestBuilder deleteRequestBuilder = client.prepareDelete(hit.getIndex(), hit.getType(), hit.getId()); + DeleteRequest deleteRequest = new DeleteRequest(hit.getIndex(), hit.getType(), hit.getId()); if (routing != null) { - deleteRequestBuilder.setRouting(routing.getValue()); + deleteRequest.routing(routing.getValue()); } - add(deleteRequestBuilder.request()); + add(deleteRequest); } String scrollId = searchResponse.getScrollId(); if (scrollId == null) { break; } - searchResponse = client.prepareSearchScroll(scrollId).setScroll(TimeValue.timeValueMinutes(5)).get(); + searchResponse = esClient.scroll(new SearchScrollRequest(scrollId).scroll(TimeValue.timeValueMinutes(5))); if (hits.length == 0) { - client.nativeClient().prepareClearScroll().addScrollId(scrollId).get(); + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); + clearScrollRequest.addScrollId(scrollId); + esClient.clearScroll(clearScrollRequest); break; } } } public void addDeletion(IndexType indexType, String id) { - add(client.prepareDelete(indexType, id).request()); + add(new DeleteRequest(indexType.getMainType().getIndex().getName(), indexType.getMainType().getType(), id)); } public void addDeletion(IndexType indexType, String id, @Nullable String routing) { - add(client.prepareDelete(indexType, id).setRouting(routing).request()); + add(new DeleteRequest(indexType.getMainType().getIndex().getName(), indexType.getMainType().getType(), id).routing(routing)); } /** @@ -191,7 +202,7 @@ public class BulkIndexer { * <p> * Note that the parameter indexType could be removed if progress logs are not needed. */ - public static IndexingResult delete(EsClient client, IndexType indexType, SearchRequestBuilder searchRequest) { + public static IndexingResult delete(EsClient client, IndexType indexType, SearchRequest searchRequest) { BulkIndexer bulk = new BulkIndexer(client, indexType, Size.REGULAR); bulk.start(); bulk.addDeletion(searchRequest); @@ -381,13 +392,14 @@ public class BulkIndexer { .setPluralLabel("requests"); this.progress.start(); Map<String, Object> temporarySettings = new HashMap<>(); - GetSettingsResponse settingsResp = bulkIndexer.client.nativeClient().admin().indices().prepareGetSettings(index).get(); + + GetSettingsResponse settingsResp = bulkIndexer.esClient.getSettings(new GetSettingsRequest()); // deactivate replicas - int initialReplicas = Integer.parseInt(settingsResp.getSetting(index, IndexMetaData.SETTING_NUMBER_OF_REPLICAS)); + int initialReplicas = Integer.parseInt(settingsResp.getSetting(index, IndexMetadata.SETTING_NUMBER_OF_REPLICAS)); if (initialReplicas > 0) { - initialSettings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, initialReplicas); - temporarySettings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0); + initialSettings.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, initialReplicas); + temporarySettings.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0); } // deactivate periodical refresh @@ -403,16 +415,16 @@ public class BulkIndexer { // optimize lucene segments and revert index settings // Optimization must be done before re-applying replicas: // http://www.elasticsearch.org/blog/performance-considerations-elasticsearch-indexing/ - bulkIndexer.client.prepareForceMerge(bulkIndexer.indexType.getMainType().getIndex().getName()).get(); + bulkIndexer.esClient.forcemerge(new ForceMergeRequest(bulkIndexer.indexType.getMainType().getIndex().getName())); updateSettings(bulkIndexer, initialSettings); this.progress.stop(); } private static void updateSettings(BulkIndexer bulkIndexer, Map<String, Object> settings) { - UpdateSettingsRequestBuilder req = bulkIndexer.client.nativeClient().admin().indices().prepareUpdateSettings(bulkIndexer.indexType.getMainType().getIndex().getName()); - req.setSettings(settings); - req.get(); + UpdateSettingsRequest req = new UpdateSettingsRequest(bulkIndexer.indexType.getMainType().getIndex().getName()); + req.settings(settings); + bulkIndexer.esClient.putSettings(req); } } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/package-info.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/ElasticsearchException.java index 408ecb94a59..00ffbdd61d3 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/package-info.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/ElasticsearchException.java @@ -17,7 +17,11 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -@ParametersAreNonnullByDefault -package org.sonar.server.es.request; +package org.sonar.server.es; -import javax.annotation.ParametersAreNonnullByDefault; +public class ElasticsearchException extends RuntimeException { + + public ElasticsearchException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClient.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClient.java index 87d32d64f5a..98941a70998 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClient.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClient.java @@ -19,159 +19,273 @@ */ package org.sonar.server.es; +import com.google.common.collect.Lists; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; import java.io.Closeable; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestBuilder; -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; -import org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequestBuilder; -import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequestBuilder; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequestBuilder; -import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequestBuilder; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; -import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; -import org.elasticsearch.action.delete.DeleteRequestBuilder; -import org.elasticsearch.action.get.GetRequestBuilder; -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchScrollRequestBuilder; -import org.elasticsearch.client.Client; +import java.io.IOException; +import java.util.Arrays; +import java.util.function.Supplier; +import org.apache.http.HttpHost; +import org.apache.http.util.EntityUtils; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; +import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.delete.DeleteResponse; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.search.ClearScrollRequest; +import org.elasticsearch.action.search.ClearScrollResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchScrollRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.Cancellable; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Requests; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.client.indices.CreateIndexResponse; +import org.elasticsearch.client.indices.GetIndexRequest; +import org.elasticsearch.client.indices.GetIndexResponse; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.Priority; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; -import org.sonar.server.es.IndexType.IndexMainType; -import org.sonar.server.es.request.ProxyClearCacheRequestBuilder; -import org.sonar.server.es.request.ProxyClusterHealthRequestBuilder; -import org.sonar.server.es.request.ProxyClusterStateRequestBuilder; -import org.sonar.server.es.request.ProxyClusterStatsRequestBuilder; -import org.sonar.server.es.request.ProxyCreateIndexRequestBuilder; -import org.sonar.server.es.request.ProxyDeleteRequestBuilder; -import org.sonar.server.es.request.ProxyGetRequestBuilder; -import org.sonar.server.es.request.ProxyIndexRequestBuilder; -import org.sonar.server.es.request.ProxyIndicesExistsRequestBuilder; -import org.sonar.server.es.request.ProxyIndicesStatsRequestBuilder; -import org.sonar.server.es.request.ProxyNodesStatsRequestBuilder; -import org.sonar.server.es.request.ProxyPutMappingRequestBuilder; -import org.sonar.server.es.request.ProxyRefreshRequestBuilder; -import org.sonar.server.es.request.ProxySearchRequestBuilder; -import org.sonar.server.es.request.ProxySearchScrollRequestBuilder; - -import static java.util.Objects.requireNonNull; +import org.sonar.api.utils.log.Profiler; +import org.sonar.server.es.response.ClusterStatsResponse; +import org.sonar.server.es.response.IndicesStatsResponse; +import org.sonar.server.es.response.NodeStatsResponse; + +import static org.sonar.server.es.EsRequestDetails.computeDetailsAsString; /** - * Facade to connect to Elasticsearch node. Handles correctly errors (logging + exceptions + * Wrapper to connect to Elasticsearch node. Handles correctly errors (logging + exceptions * with context) and profiling of requests. */ public class EsClient implements Closeable { + private final RestHighLevelClient restHighLevelClient; + private final Gson gson; public static final Logger LOGGER = Loggers.get("es"); - private final Client nativeClient; + public EsClient(HttpHost... hosts) { + this(new MinimalRestHighLevelClient(hosts)); + } + + EsClient(RestHighLevelClient restHighLevelClient) { + this.restHighLevelClient = restHighLevelClient; + this.gson = new GsonBuilder().create(); + } - public EsClient(Client nativeClient) { - this.nativeClient = requireNonNull(nativeClient); + public BulkResponse bulk(BulkRequest bulkRequest) { + return execute(() -> restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT)); } - public EsClient() { - this.nativeClient = null; + public Cancellable bulkAsync(BulkRequest bulkRequest, ActionListener<BulkResponse> listener) { + return restHighLevelClient.bulkAsync(bulkRequest, RequestOptions.DEFAULT, listener); } - public RefreshRequestBuilder prepareRefresh(Index index) { - return new ProxyRefreshRequestBuilder(nativeClient()).setIndices(index.getName()); + public static SearchRequest prepareSearch(String indexName) { + return Requests.searchRequest(indexName); } - public IndicesStatsRequestBuilder prepareStats() { - return new ProxyIndicesStatsRequestBuilder(nativeClient()); + public static SearchRequest prepareSearch(IndexType.IndexMainType mainType) { + return Requests.searchRequest(mainType.getIndex().getName()).types(mainType.getType()); } - public IndicesStatsRequestBuilder prepareStats(Index index) { - return new ProxyIndicesStatsRequestBuilder(nativeClient()).setIndices(index.getName()); + public static SearchRequest prepareSearch(String index, String type) { + return Requests.searchRequest(index).types(type); } - public NodesStatsRequestBuilder prepareNodesStats(String... nodesIds) { - return new ProxyNodesStatsRequestBuilder(nativeClient()).setNodesIds(nodesIds); + public SearchResponse search(SearchRequest searchRequest) { + return execute(() -> restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT), + () -> computeDetailsAsString(searchRequest)); } - public ClusterStatsRequestBuilder prepareClusterStats() { - return new ProxyClusterStatsRequestBuilder(nativeClient()); + public SearchResponse scroll(SearchScrollRequest searchScrollRequest) { + return execute(() -> restHighLevelClient.scroll(searchScrollRequest, RequestOptions.DEFAULT), + () -> computeDetailsAsString(searchScrollRequest)); } - public ClusterStateRequestBuilder prepareState() { - return new ProxyClusterStateRequestBuilder(nativeClient()); + public ClearScrollResponse clearScroll(ClearScrollRequest clearScrollRequest) { + return execute(() -> restHighLevelClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT)); } - public ClusterHealthRequestBuilder prepareHealth() { - return new ProxyClusterHealthRequestBuilder(nativeClient()); + public DeleteResponse delete(DeleteRequest deleteRequest) { + return execute(() -> restHighLevelClient.delete(deleteRequest, RequestOptions.DEFAULT), + () -> computeDetailsAsString(deleteRequest)); } - public void waitForStatus(ClusterHealthStatus status) { - prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForStatus(status).get(); + public RefreshResponse refresh(Index... indices) { + RefreshRequest refreshRequest = new RefreshRequest() + .indices(Arrays.stream(indices).map(Index::getName).toArray(String[]::new)); + return execute(() -> restHighLevelClient.indices().refresh(refreshRequest, RequestOptions.DEFAULT), + () -> computeDetailsAsString(refreshRequest)); } - public IndicesExistsRequestBuilder prepareIndicesExist(Index index) { - return new ProxyIndicesExistsRequestBuilder(nativeClient(), index.getName()); + public ForceMergeResponse forcemerge(ForceMergeRequest forceMergeRequest) { + return execute(() -> restHighLevelClient.indices().forcemerge(forceMergeRequest, RequestOptions.DEFAULT)); } - public CreateIndexRequestBuilder prepareCreate(Index index) { - return new ProxyCreateIndexRequestBuilder(nativeClient(), index.getName()); + public AcknowledgedResponse putSettings(UpdateSettingsRequest req) { + return execute(() -> restHighLevelClient.indices().putSettings(req, RequestOptions.DEFAULT)); } - public PutMappingRequestBuilder preparePutMapping(Index index) { - return new ProxyPutMappingRequestBuilder(nativeClient()).setIndices(index.getName()); + public ClearIndicesCacheResponse clearCache(ClearIndicesCacheRequest request) { + return execute(() -> restHighLevelClient.indices().clearCache(request, RequestOptions.DEFAULT), + () -> computeDetailsAsString(request)); } - public SearchRequestBuilder prepareSearch(Index index) { - return new ProxySearchRequestBuilder(nativeClient()).setIndices(index.getName()); + public IndexResponse index(IndexRequest indexRequest) { + return execute(() -> restHighLevelClient.index(indexRequest, RequestOptions.DEFAULT), + () -> computeDetailsAsString(indexRequest)); } - public SearchRequestBuilder - prepareSearch(IndexMainType indexType) { - return new ProxySearchRequestBuilder(nativeClient()) - .setIndices(indexType.getIndex().getName()) - .setTypes(indexType.getType()); + public GetResponse get(GetRequest request) { + return execute(() -> restHighLevelClient.get(request, RequestOptions.DEFAULT), + () -> computeDetailsAsString(request)); } - public SearchScrollRequestBuilder prepareSearchScroll(String scrollId) { - return new ProxySearchScrollRequestBuilder(scrollId, nativeClient()); + public GetIndexResponse getIndex(GetIndexRequest getRequest) { + return execute(() -> restHighLevelClient.indices().get(getRequest, RequestOptions.DEFAULT)); } - public GetRequestBuilder prepareGet(IndexType indexType, String id) { - IndexMainType mainType = indexType.getMainType(); - return new ProxyGetRequestBuilder(nativeClient()).setIndex(mainType.getIndex().getName()).setType(mainType.getType()).setId(id); + public boolean indexExists(GetIndexRequest getIndexRequest) { + return execute(() -> restHighLevelClient.indices().exists(getIndexRequest, RequestOptions.DEFAULT), + () -> computeDetailsAsString(getIndexRequest)); } - public DeleteRequestBuilder prepareDelete(IndexType indexType, String id) { - IndexMainType mainType = indexType.getMainType(); - return new ProxyDeleteRequestBuilder(nativeClient(), mainType.getIndex().getName()).setType(mainType.getType()).setId(id); + public CreateIndexResponse create(CreateIndexRequest createIndexRequest) { + return execute(() -> restHighLevelClient.indices().create(createIndexRequest, RequestOptions.DEFAULT), + () -> computeDetailsAsString(createIndexRequest)); } - DeleteRequestBuilder prepareDelete(String index, String type, String id) { - return new ProxyDeleteRequestBuilder(nativeClient(), index).setType(type).setId(id); + public AcknowledgedResponse deleteIndex(DeleteIndexRequest deleteIndexRequest) { + return execute(() -> restHighLevelClient.indices().delete(deleteIndexRequest, RequestOptions.DEFAULT)); } - public IndexRequestBuilder prepareIndex(IndexType indexType) { - IndexMainType mainType = indexType.getMainType(); - return new ProxyIndexRequestBuilder(nativeClient()).setIndex(mainType.getIndex().getName()).setType(mainType.getType()); + public AcknowledgedResponse putMapping(PutMappingRequest request) { + return execute(() -> restHighLevelClient.indices().putMapping(request, RequestOptions.DEFAULT), + () -> computeDetailsAsString(request)); } - public ForceMergeRequestBuilder prepareForceMerge(String indexName) { - // TODO add proxy for profiling - return nativeClient().admin().indices().prepareForceMerge(indexName) - .setMaxNumSegments(1); + public ClusterHealthResponse clusterHealth(ClusterHealthRequest clusterHealthRequest) { + return execute(() -> restHighLevelClient.cluster().health(clusterHealthRequest, RequestOptions.DEFAULT), + () -> computeDetailsAsString(clusterHealthRequest)); } - public ClearIndicesCacheRequestBuilder prepareClearCache(String... indices) { - return new ProxyClearCacheRequestBuilder(nativeClient()).setIndices(indices); + public void waitForStatus(ClusterHealthStatus clusterHealthStatus) { + clusterHealth(new ClusterHealthRequest().waitForEvents(Priority.LANGUID).waitForStatus(clusterHealthStatus)); } - public Client nativeClient() { - return nativeClient; + // https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-nodes-stats.html + public NodeStatsResponse nodesStats() { + return execute(() -> { + Request request = new Request("GET", "/_nodes/stats/fs,process,jvm,indices,breaker"); + Response response = restHighLevelClient.getLowLevelClient().performRequest(request); + return NodeStatsResponse.toNodeStatsResponse(gson.fromJson(EntityUtils.toString(response.getEntity()), JsonObject.class)); + }); + } + + // https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-stats.html + public IndicesStatsResponse indicesStats(String... indices) { + return execute(() -> { + Request request = new Request("GET", "/" + (indices.length > 0 ? (String.join(",", indices) + "/") : "") + "_stats"); + request.addParameter("level", "shards"); + Response response = restHighLevelClient.getLowLevelClient().performRequest(request); + return IndicesStatsResponse.toIndicesStatsResponse(gson.fromJson(EntityUtils.toString(response.getEntity()), JsonObject.class)); + }, () -> computeDetailsAsString(indices)); + } + + // https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-stats.html + public ClusterStatsResponse clusterStats() { + return execute(() -> { + Request request = new Request("GET", "/_cluster/stats"); + Response response = restHighLevelClient.getLowLevelClient().performRequest(request); + return ClusterStatsResponse.toClusterStatsResponse(gson.fromJson(EntityUtils.toString(response.getEntity()), JsonObject.class)); + }); + } + + public GetSettingsResponse getSettings(GetSettingsRequest getSettingsRequest) { + return execute(() -> restHighLevelClient.indices().getSettings(getSettingsRequest, RequestOptions.DEFAULT)); + } + + public GetMappingsResponse getMapping(GetMappingsRequest getMappingsRequest) { + return execute(() -> restHighLevelClient.indices().getMapping(getMappingsRequest, RequestOptions.DEFAULT)); } @Override public void close() { - nativeClient.close(); + try { + restHighLevelClient.close(); + } catch (IOException e) { + throw new ElasticsearchException("Could not close ES Rest high level client", e); + } } + + /** + * Internal usage only + * + * @return native ES client object + */ + RestHighLevelClient nativeClient() { + return restHighLevelClient; + } + + static class MinimalRestHighLevelClient extends RestHighLevelClient { + + public MinimalRestHighLevelClient(HttpHost... hosts) { + super(RestClient.builder(hosts)); + } + + MinimalRestHighLevelClient(RestClient restClient) { + super(restClient, RestClient::close, Lists.newArrayList()); + } + } + + <R> R execute(EsRequestExecutor<R> executor) { + return execute(executor, () -> ""); + } + + <R> R execute(EsRequestExecutor<R> executor, Supplier<String> requestDetails) { + Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); + try { + return executor.execute(); + } catch (Exception e) { + throw new ElasticsearchException("Fail to execute es request" + requestDetails.get(), e); + } finally { + if (profiler.isTraceEnabled()) { + profiler.stopTrace(requestDetails.get()); + } + } + } + + @FunctionalInterface + interface EsRequestExecutor<R> { + R execute() throws IOException; + } + } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClientProvider.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClientProvider.java index 4e7a52ad257..f84189a3262 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClientProvider.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClientProvider.java @@ -20,20 +20,14 @@ package org.sonar.server.es; import com.google.common.net.HostAndPort; -import io.netty.util.ThreadDeathWatcher; -import io.netty.util.concurrent.GlobalEventExecutor; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Arrays; -import java.util.concurrent.TimeUnit; +import java.util.Collections; +import java.util.List; import java.util.stream.Collectors; -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.network.NetworkModule; +import org.apache.http.HttpHost; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.join.ParentJoinPlugin; -import org.elasticsearch.percolator.PercolatorPlugin; -import org.elasticsearch.transport.Netty4Plugin; import org.picocontainer.injectors.ProviderAdapter; import org.sonar.api.ce.ComputeEngineSide; import org.sonar.api.config.Configuration; @@ -42,7 +36,6 @@ import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; import org.sonar.process.cluster.NodeType; -import static java.util.Collections.unmodifiableList; import static org.sonar.process.ProcessProperties.Property.CLUSTER_ENABLED; import static org.sonar.process.ProcessProperties.Property.CLUSTER_NAME; import static org.sonar.process.ProcessProperties.Property.CLUSTER_NODE_TYPE; @@ -68,58 +61,41 @@ public class EsClientProvider extends ProviderAdapter { boolean clusterEnabled = config.getBoolean(CLUSTER_ENABLED.getKey()).orElse(false); boolean searchNode = !clusterEnabled || SEARCH.equals(NodeType.parse(config.get(CLUSTER_NODE_TYPE.getKey()).orElse(null))); - final TransportClient nativeClient = new MinimalTransportClient(esSettings.build()); + List<HttpHost> httpHosts; if (clusterEnabled && !searchNode) { - esSettings.put("client.transport.sniff", true); - Arrays.stream(config.getStringArray(CLUSTER_SEARCH_HOSTS.getKey())) - .map(HostAndPort::fromString) - .forEach(h -> addHostToClient(h, nativeClient)); - LOGGER.info("Connected to remote Elasticsearch: [{}]", displayedAddresses(nativeClient)); + httpHosts = getHttpHosts(config); + + LOGGER.info("Connected to remote Elasticsearch: [{}]", displayedAddresses(httpHosts)); } else { + //defaults provided in: + // * in org.sonar.process.ProcessProperties.Property.SEARCH_HOST + // * in org.sonar.process.ProcessProperties.Property.SEARCH_PORT HostAndPort host = HostAndPort.fromParts(config.get(SEARCH_HOST.getKey()).get(), config.getInt(SEARCH_PORT.getKey()).get()); - addHostToClient(host, nativeClient); - LOGGER.info("Connected to local Elasticsearch: [{}]", displayedAddresses(nativeClient)); + httpHosts = Collections.singletonList(toHttpHost(host)); + LOGGER.info("Connected to local Elasticsearch: [{}]", displayedAddresses(httpHosts)); } - cache = new EsClient(nativeClient); + cache = new EsClient(httpHosts.toArray(new HttpHost[0])); } return cache; } - private static void addHostToClient(HostAndPort host, TransportClient client) { + private static List<HttpHost> getHttpHosts(Configuration config) { + return Arrays.stream(config.getStringArray(CLUSTER_SEARCH_HOSTS.getKey())) + .map(HostAndPort::fromString) + .map(EsClientProvider::toHttpHost) + .collect(Collectors.toList()); + } + + private static HttpHost toHttpHost(HostAndPort host) { try { - client.addTransportAddress(new TransportAddress(InetAddress.getByName(host.getHost()), host.getPortOrDefault(9001))); + return new HttpHost(InetAddress.getByName(host.getHost()), host.getPortOrDefault(9001)); } catch (UnknownHostException e) { throw new IllegalStateException("Can not resolve host [" + host + "]", e); } } - private static String displayedAddresses(TransportClient nativeClient) { - return nativeClient.transportAddresses().stream().map(TransportAddress::toString).collect(Collectors.joining(", ")); - } - - static class MinimalTransportClient extends TransportClient { - - MinimalTransportClient(Settings settings) { - super(settings, unmodifiableList(Arrays.asList(Netty4Plugin.class, PercolatorPlugin.class, ParentJoinPlugin.class))); - } - - @Override - public void close() { - super.close(); - if (!NetworkModule.TRANSPORT_TYPE_SETTING.exists(settings) - || NetworkModule.TRANSPORT_TYPE_SETTING.get(settings).equals(Netty4Plugin.NETTY_TRANSPORT_NAME)) { - try { - GlobalEventExecutor.INSTANCE.awaitInactivity(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - try { - ThreadDeathWatcher.awaitInactivity(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - } + private static String displayedAddresses(List<HttpHost> httpHosts) { + return httpHosts.stream().map(HttpHost::toString).collect(Collectors.joining(", ")); } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/EsRequestDetails.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsRequestDetails.java new file mode 100644 index 00000000000..0b8e19b4083 --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsRequestDetails.java @@ -0,0 +1,168 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es; + +import java.util.Arrays; +import org.apache.commons.lang.StringUtils; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchScrollRequest; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.client.indices.GetIndexRequest; + +final class EsRequestDetails { + private static final String ON_INDICES_MESSAGE = " on indices '%s'"; + private static final String ON_TYPE_MESSAGE = " on type '%s'"; + + private EsRequestDetails() { + // this is utility class only + } + + static String computeDetailsAsString(SearchRequest searchRequest) { + StringBuilder message = new StringBuilder(); + message.append(String.format("ES search request '%s'", searchRequest)); + if (searchRequest.indices().length > 0) { + message.append(String.format(ON_INDICES_MESSAGE, Arrays.toString(searchRequest.indices()))); + } + if (searchRequest.types().length > 0) { + message.append(String.format(" on types '%s'", Arrays.toString(searchRequest.types()))); + } + return message.toString(); + } + + public static String computeDetailsAsString(SearchScrollRequest searchScrollRequest) { + return String.format("ES search scroll request for scroll id '%s'", searchScrollRequest.scroll()); + } + + static String computeDetailsAsString(DeleteRequest deleteRequest) { + return new StringBuilder() + .append("ES delete request of doc ") + .append(deleteRequest.id()) + .append(" in index ") + .append(deleteRequest.index()) + .append("/") + .append(deleteRequest.type()) + .toString(); + } + + static String computeDetailsAsString(RefreshRequest refreshRequest) { + StringBuilder message = new StringBuilder(); + message.append("ES refresh request"); + if (refreshRequest.indices().length > 0) { + message.append(String.format(ON_INDICES_MESSAGE, StringUtils.join(refreshRequest.indices(), ","))); + } + return message.toString(); + } + + static String computeDetailsAsString(ClearIndicesCacheRequest request) { + StringBuilder message = new StringBuilder(); + message.append("ES clear cache request"); + if (request.indices().length > 0) { + message.append(String.format(ON_INDICES_MESSAGE, StringUtils.join(request.indices(), ","))); + } + String[] fields = request.fields(); + if (fields != null && fields.length > 0) { + message.append(String.format(" on fields '%s'", StringUtils.join(fields, ","))); + } + if (request.queryCache()) { + message.append(" with filter cache"); + } + if (request.fieldDataCache()) { + message.append(" with field data cache"); + } + if (request.requestCache()) { + message.append(" with request cache"); + } + return message.toString(); + } + + static String computeDetailsAsString(IndexRequest indexRequest) { + return new StringBuilder().append("ES index request") + .append(String.format(" for key '%s'", indexRequest.id())) + .append(String.format(" on index '%s'", indexRequest.index())) + .append(String.format(ON_TYPE_MESSAGE, indexRequest.type())) + .toString(); + } + + static String computeDetailsAsString(GetRequest request) { + return new StringBuilder().append("ES get request") + .append(String.format(" for key '%s'", request.id())) + .append(String.format(" on index '%s'", request.index())) + .append(String.format(ON_TYPE_MESSAGE, request.type())) + .toString(); + } + + static String computeDetailsAsString(GetIndexRequest getIndexRequest) { + StringBuilder message = new StringBuilder(); + message.append("ES indices exists request"); + if (getIndexRequest.indices().length > 0) { + message.append(String.format(ON_INDICES_MESSAGE, StringUtils.join(getIndexRequest.indices(), ","))); + } + return message.toString(); + } + + static String computeDetailsAsString(CreateIndexRequest createIndexRequest) { + return String.format("ES create index '%s'", createIndexRequest.index()); + } + + static String computeDetailsAsString(PutMappingRequest request) { + StringBuilder message = new StringBuilder(); + message.append("ES put mapping request"); + if (request.indices().length > 0) { + message.append(String.format(ON_INDICES_MESSAGE, StringUtils.join(request.indices(), ","))); + } + String type = request.type(); + if (type != null) { + message.append(String.format(ON_TYPE_MESSAGE, type)); + } + String source = request.source(); + if (source != null) { + message.append(String.format(" with source '%s'", source)); + } + + return message.toString(); + } + + static String computeDetailsAsString(ClusterHealthRequest clusterHealthRequest) { + StringBuilder message = new StringBuilder(); + message.append("ES cluster health request"); + String[] indices = clusterHealthRequest.indices(); + if (indices != null && indices.length > 0) { + message.append(String.format(ON_INDICES_MESSAGE, StringUtils.join(indices, ","))); + } + return message.toString(); + } + + static String computeDetailsAsString(String... indices) { + StringBuilder message = new StringBuilder(); + message.append("ES indices stats request"); + if (indices.length > 0) { + message.append(String.format(ON_INDICES_MESSAGE, StringUtils.join(indices, ","))); + } + return message.toString(); + } + +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/EsUtils.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsUtils.java index cc6692ff426..e99a61b168d 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/EsUtils.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsUtils.java @@ -33,13 +33,13 @@ import java.util.function.Function; import java.util.regex.Pattern; import javax.annotation.CheckForNull; import javax.annotation.Nullable; -import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchScrollRequestBuilder; +import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortOrder; import org.joda.time.format.ISODateTimeFormat; import org.sonar.core.util.stream.MoreCollectors; @@ -101,8 +101,8 @@ public class EsUtils { * Optimize scolling, by specifying document sorting. * See https://www.elastic.co/guide/en/elasticsearch/reference/2.4/search-request-scroll.html#search-request-scroll */ - public static void optimizeScrollRequest(SearchRequestBuilder esSearch) { - esSearch.addSort("_doc", SortOrder.ASC); + public static void optimizeScrollRequest(SearchSourceBuilder esSearch) { + esSearch.sort("_doc", SortOrder.ASC); } /** @@ -136,9 +136,9 @@ public class EsUtils { @Override public boolean hasNext() { if (hits.isEmpty()) { - SearchScrollRequestBuilder esRequest = esClient.prepareSearchScroll(scrollId) - .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)); - Collections.addAll(hits, esRequest.get().getHits().getHits()); + SearchScrollRequest esRequest = new SearchScrollRequest(scrollId) + .scroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)); + Collections.addAll(hits, esClient.scroll(esRequest).getHits().getHits()); } return !hits.isEmpty(); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/Facets.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/Facets.java index 3aa00c816da..27e7d089ae8 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/Facets.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/Facets.java @@ -38,7 +38,7 @@ import org.elasticsearch.search.aggregations.bucket.filter.Filter; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.missing.Missing; import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.aggregations.metrics.sum.Sum; +import org.elasticsearch.search.aggregations.metrics.Sum; import static org.sonar.api.utils.DateUtils.parseDateTime; import static org.sonarqube.ws.client.issue.IssuesWsParameters.FACET_MODE_EFFORT; diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/SearchIdResult.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/SearchIdResult.java index 6313b59eeca..c369f0759a9 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/SearchIdResult.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/SearchIdResult.java @@ -36,7 +36,7 @@ public class SearchIdResult<ID> { public SearchIdResult(SearchResponse response, Function<String, ID> converter, TimeZone timeZone) { this.facets = new Facets(response, timeZone); - this.total = response.getHits().getTotalHits(); + this.total = response.getHits().getTotalHits().value; this.uuids = convertToIds(response.getHits(), converter); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/SearchResult.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/SearchResult.java index ddee72eb3be..996ba9afd1d 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/SearchResult.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/SearchResult.java @@ -34,7 +34,7 @@ public class SearchResult<DOC extends BaseDoc> { public SearchResult(SearchResponse response, Function<Map<String, Object>, DOC> converter, TimeZone timeZone) { this.facets = new Facets(response, timeZone); - this.total = response.getHits().getTotalHits(); + this.total = response.getHits().getTotalHits().value; this.docs = EsUtils.convertToDocs(response.getHits(), converter); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndexImpl.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndexImpl.java index a8646bad84b..41613aa45e4 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndexImpl.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndexImpl.java @@ -20,8 +20,9 @@ package org.sonar.server.es.metadata; import java.util.Optional; -import org.elasticsearch.action.get.GetRequestBuilder; +import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.common.document.DocumentField; import org.sonar.server.es.EsClient; import org.sonar.server.es.Index; @@ -90,9 +91,8 @@ public class MetadataIndexImpl implements MetadataIndex { } private Optional<String> getMetadata(String id) { - GetRequestBuilder request = esClient.prepareGet(TYPE_METADATA, id) - .setStoredFields(MetadataIndexDefinition.FIELD_VALUE); - GetResponse response = request.get(); + GetResponse response = esClient.get(new GetRequest(TYPE_METADATA.getIndex().getName(), TYPE_METADATA.getType(), id) + .storedFields(MetadataIndexDefinition.FIELD_VALUE)); if (response.isExists()) { DocumentField field = response.getField(MetadataIndexDefinition.FIELD_VALUE); return Optional.of(field.getValue()); @@ -101,10 +101,8 @@ public class MetadataIndexImpl implements MetadataIndex { } private void setMetadata(String id, String value) { - esClient.prepareIndex(TYPE_METADATA) - .setId(id) - .setSource(MetadataIndexDefinition.FIELD_VALUE, value) - .setRefreshPolicy(REFRESH_IMMEDIATE) - .get(); + esClient.index(new IndexRequest(TYPE_METADATA.getIndex().getName(), TYPE_METADATA.getType(), id) + .source(MetadataIndexDefinition.FIELD_VALUE, value) + .setRefreshPolicy(REFRESH_IMMEDIATE)); } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/DefaultIndexSettings.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/DefaultIndexSettings.java index bfc82efbe15..53ff8d5ec23 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/DefaultIndexSettings.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/DefaultIndexSettings.java @@ -21,7 +21,7 @@ package org.sonar.server.es.newindex; import java.util.Arrays; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; -import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; public class DefaultIndexSettings { @@ -82,7 +82,7 @@ public class DefaultIndexSettings { public static Settings.Builder defaults() { Settings.Builder builder = Settings.builder() - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put("index.refresh_interval", "30s"); Arrays.stream(DefaultIndexSettingsElement.values()) diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/DefaultIndexSettingsElement.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/DefaultIndexSettingsElement.java index ad3517a70e7..c6c5c00af5f 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/DefaultIndexSettingsElement.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/DefaultIndexSettingsElement.java @@ -78,7 +78,7 @@ public enum DefaultIndexSettingsElement { @Override protected void setup() { - set(TYPE, "nGram"); + set(TYPE, "ngram"); set(MIN_GRAM, MINIMUM_NGRAM_LENGTH); set(MAX_GRAM, MAXIMUM_NGRAM_LENGTH); setList("token_chars", "letter", "digit", "punctuation", "symbol"); @@ -91,7 +91,7 @@ public enum DefaultIndexSettingsElement { @Override protected void setup() { - set(TYPE, "nGram"); + set(TYPE, "ngram"); set(MIN_GRAM, MINIMUM_NGRAM_LENGTH); set(MAX_GRAM, MAXIMUM_NGRAM_LENGTH); setList("token_chars", "letter", "digit", "punctuation", "symbol"); @@ -101,7 +101,7 @@ public enum DefaultIndexSettingsElement { @Override protected void setup() { - set(TYPE, "edgeNGram"); + set(TYPE, "edge_ngram"); set(MIN_GRAM, MINIMUM_NGRAM_LENGTH); set(MAX_GRAM, MAXIMUM_NGRAM_LENGTH); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/NewIndex.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/NewIndex.java index 74a9bf2eec8..c55fde28622 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/NewIndex.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/NewIndex.java @@ -26,7 +26,7 @@ import java.util.Map; import java.util.TreeMap; import java.util.stream.Stream; import javax.annotation.CheckForNull; -import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.sonar.api.config.Configuration; import org.sonar.server.es.Index; @@ -63,8 +63,8 @@ public abstract class NewIndex<T extends NewIndex<T>> { .orElse(settingsConfiguration.getDefaultNbOfShards()); int replicas = clusterMode ? config.getInt(SEARCH_REPLICAS.getKey()).orElse(1) : 0; - settings.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, shards); - settings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, replicas); + settings.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, shards); + settings.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, replicas); settings.put("index.max_ngram_diff", DefaultIndexSettings.MAXIMUM_NGRAM_LENGTH - DefaultIndexSettings.MINIMUM_NGRAM_LENGTH); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClearCacheRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClearCacheRequestBuilder.java deleted file mode 100644 index 6cb107a4eb5..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClearCacheRequestBuilder.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheAction; -import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequestBuilder; -import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyClearCacheRequestBuilder extends ClearIndicesCacheRequestBuilder { - - public ProxyClearCacheRequestBuilder(Client client) { - super(client.admin().indices(), ClearIndicesCacheAction.INSTANCE); - } - - @Override - public ClearIndicesCacheResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public ClearIndicesCacheResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ClearIndicesCacheResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<ClearIndicesCacheResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder(); - message.append("ES clear cache request"); - if (request.indices().length > 0) { - message.append(String.format(" on indices '%s'", StringUtils.join(request.indices(), ","))); - } - String[] fields = request.fields(); - if (fields != null && fields.length > 0) { - message.append(String.format(" on fields '%s'", StringUtils.join(fields, ","))); - } - if (request.queryCache()) { - message.append(" with filter cache"); - } - if (request.fieldDataCache()) { - message.append(" with field data cache"); - } - if (request.requestCache()) { - message.append(" with request cache"); - } - return message.toString(); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClusterHealthRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClusterHealthRequestBuilder.java deleted file mode 100644 index d616fc0f03d..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClusterHealthRequestBuilder.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyClusterHealthRequestBuilder extends ClusterHealthRequestBuilder { - - public ProxyClusterHealthRequestBuilder(Client client) { - super(client.admin().cluster(), ClusterHealthAction.INSTANCE); - } - - @Override - public ClusterHealthResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public ClusterHealthResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ClusterHealthResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<ClusterHealthResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder(); - message.append("ES cluster health request"); - String[] indices = request.indices(); - if (indices != null && indices.length > 0) { - message.append(String.format(" on indices '%s'", StringUtils.join(indices, ","))); - } - return message.toString(); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClusterStateRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClusterStateRequestBuilder.java deleted file mode 100644 index 6a19906b134..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClusterStateRequestBuilder.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; -import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyClusterStateRequestBuilder extends ClusterStateRequestBuilder { - - public ProxyClusterStateRequestBuilder(Client client) { - super(client.admin().cluster(), ClusterStateAction.INSTANCE); - } - - @Override - public ClusterStateResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public ClusterStateResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ClusterStateResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<ClusterStateResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder(); - message.append("ES cluster state request"); - if (request.indices().length > 0) { - message.append(String.format(" on indices '%s'", StringUtils.join(request.indices(), ","))); - } - return message.toString(); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClusterStatsRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClusterStatsRequestBuilder.java deleted file mode 100644 index 96f97e21bb4..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClusterStatsRequestBuilder.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.admin.cluster.stats.ClusterStatsAction; -import org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequestBuilder; -import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyClusterStatsRequestBuilder extends ClusterStatsRequestBuilder { - - public ProxyClusterStatsRequestBuilder(Client client) { - super(client.admin().cluster(), ClusterStatsAction.INSTANCE); - } - - @Override - public ClusterStatsResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public ClusterStatsResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ClusterStatsResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<ClusterStatsResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder(); - message.append("ES cluster stats request"); - if (request.nodesIds() != null) { - message.append(String.format(" on nodes '%s'", StringUtils.join(request.nodesIds(), ","))); - } - return message.toString(); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilder.java deleted file mode 100644 index 6de17fc23bd..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilder.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import java.io.IOException; -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.admin.indices.create.CreateIndexAction; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyCreateIndexRequestBuilder extends CreateIndexRequestBuilder { - - private final String index; - - public ProxyCreateIndexRequestBuilder(Client client, String index) { - super(client.admin().indices(), CreateIndexAction.INSTANCE, index); - this.index = index; - } - - @Override - public CreateIndexResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public CreateIndexResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public CreateIndexResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<CreateIndexResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - public String toJson() { - try { - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject() - .field("settings") - .startObject(); - request.settings().toXContent(builder, ToXContent.EMPTY_PARAMS); - builder.endObject().endObject(); - builder.prettyPrint(); - return builder.toString(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public String toString() { - return String.format("ES create index '%s'", index); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyDeleteRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyDeleteRequestBuilder.java deleted file mode 100644 index 2688370d747..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyDeleteRequestBuilder.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.delete.DeleteAction; -import org.elasticsearch.action.delete.DeleteRequestBuilder; -import org.elasticsearch.action.delete.DeleteResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyDeleteRequestBuilder extends DeleteRequestBuilder { - - public ProxyDeleteRequestBuilder(Client client, String index) { - super(client, DeleteAction.INSTANCE, index); - } - - @Override - public DeleteResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public DeleteResponse get(TimeValue timeout) { - throw new UnsupportedOperationException("Not yet implemented"); - } - - @Override - public DeleteResponse get(String timeout) { - throw new UnsupportedOperationException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<DeleteResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder(); - message - .append("ES delete request of doc ") - .append(request.id()) - .append(" in index ") - .append(request.index()) - .append("/") - .append(request.type()); - return message.toString(); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyGetRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyGetRequestBuilder.java deleted file mode 100644 index 98c4a9b767a..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyGetRequestBuilder.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.get.GetAction; -import org.elasticsearch.action.get.GetRequestBuilder; -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyGetRequestBuilder extends GetRequestBuilder { - - public ProxyGetRequestBuilder(Client client) { - super(client, GetAction.INSTANCE); - } - - @Override - public GetResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public GetResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public GetResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<GetResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder().append("ES get request"); - message.append(String.format(" for key '%s'", request.id())); - message.append(String.format(" on index '%s'", request.index())); - message.append(String.format(" on type '%s'", request.type())); - return message.toString(); - } - -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyIndexRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyIndexRequestBuilder.java deleted file mode 100644 index a34165a026c..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyIndexRequestBuilder.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.index.IndexAction; -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.index.IndexResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyIndexRequestBuilder extends IndexRequestBuilder { - - public ProxyIndexRequestBuilder(Client client) { - super(client, IndexAction.INSTANCE); - } - - @Override - public IndexResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public IndexResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public IndexResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<IndexResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder().append("ES index request"); - message.append(String.format(" for key '%s'", request.id())); - message.append(String.format(" on index '%s'", request.index())); - message.append(String.format(" on type '%s'", request.type())); - return message.toString(); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyIndicesExistsRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyIndicesExistsRequestBuilder.java deleted file mode 100644 index 14e27d25f91..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyIndicesExistsRequestBuilder.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequestBuilder; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyIndicesExistsRequestBuilder extends IndicesExistsRequestBuilder { - - public ProxyIndicesExistsRequestBuilder(Client client, String... indices) { - super(client.admin().indices(), IndicesExistsAction.INSTANCE, indices); - } - - @Override - public IndicesExistsResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public IndicesExistsResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public IndicesExistsResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<IndicesExistsResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder(); - message.append("ES indices exists request"); - if (request.indices().length > 0) { - message.append(String.format(" on indices '%s'", StringUtils.join(request.indices(), ","))); - } - return message.toString(); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilder.java deleted file mode 100644 index 798c5724ef4..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilder.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyIndicesStatsRequestBuilder extends IndicesStatsRequestBuilder { - - public ProxyIndicesStatsRequestBuilder(Client client) { - super(client.admin().indices(), IndicesStatsAction.INSTANCE); - } - - @Override - public IndicesStatsResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public IndicesStatsResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public IndicesStatsResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<IndicesStatsResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder(); - message.append("ES indices stats request"); - String[] indices = request.indices(); - if (indices != null && indices.length > 0) { - message.append(String.format(" on indices '%s'", StringUtils.join(indices, ","))); - } - return message.toString(); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyNodesStatsRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyNodesStatsRequestBuilder.java deleted file mode 100644 index 76d16fa02b2..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyNodesStatsRequestBuilder.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsAction; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestBuilder; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyNodesStatsRequestBuilder extends NodesStatsRequestBuilder { - - public ProxyNodesStatsRequestBuilder(Client client) { - super(client.admin().cluster(), NodesStatsAction.INSTANCE); - } - - @Override - public NodesStatsResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public NodesStatsResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public NodesStatsResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<NodesStatsResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder(); - message.append("ES nodes stats request"); - if (request.nodesIds().length > 0) { - message.append(String.format(" on nodes '%s'", StringUtils.join(request.nodesIds(), ","))); - } - return message.toString(); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilder.java deleted file mode 100644 index 4c74b7a9de5..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilder.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.ActionFuture; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyPutMappingRequestBuilder extends PutMappingRequestBuilder { - - public ProxyPutMappingRequestBuilder(Client client) { - super(client.admin().indices(), PutMappingAction.INSTANCE); - } - - @Override - public AcknowledgedResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public AcknowledgedResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public AcknowledgedResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ActionFuture<AcknowledgedResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder(); - message.append("ES put mapping request"); - if (request.indices().length > 0) { - message.append(String.format(" on indices '%s'", StringUtils.join(request.indices(), ","))); - } - String type = request.type(); - if (type != null) { - message.append(String.format(" on type '%s'", type)); - } - String source = request.source(); - if (source != null) { - message.append(String.format(" with source '%s'", source)); - } - - return message.toString(); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyRefreshRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyRefreshRequestBuilder.java deleted file mode 100644 index af72e58976f..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyRefreshRequestBuilder.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.admin.indices.refresh.RefreshAction; -import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder; -import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxyRefreshRequestBuilder extends RefreshRequestBuilder { - - public ProxyRefreshRequestBuilder(Client client) { - super(client.admin().indices(), RefreshAction.INSTANCE); - } - - @Override - public RefreshResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public RefreshResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public RefreshResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<RefreshResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder(); - message.append("ES refresh request"); - if (request.indices().length > 0) { - message.append(String.format(" on indices '%s'", StringUtils.join(request.indices(), ","))); - } - return message.toString(); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxySearchRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxySearchRequestBuilder.java deleted file mode 100644 index 8250b2f58d3..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxySearchRequestBuilder.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import java.util.Arrays; -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxySearchRequestBuilder extends SearchRequestBuilder { - - public ProxySearchRequestBuilder(Client client) { - super(client, SearchAction.INSTANCE); - } - - @Override - public SearchResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public SearchResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public SearchResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<SearchResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder(); - message.append(String.format("ES search request '%s'", super.request)); - if (request.indices().length > 0) { - message.append(String.format(" on indices '%s'", Arrays.toString(request.indices()))); - } - if (request.types().length > 0) { - message.append(String.format(" on types '%s'", Arrays.toString(request.types()))); - } - return message.toString(); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilder.java deleted file mode 100644 index 1f56355114b..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilder.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchScrollAction; -import org.elasticsearch.action.search.SearchScrollRequestBuilder; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -public class ProxySearchScrollRequestBuilder extends SearchScrollRequestBuilder { - - public ProxySearchScrollRequestBuilder(String scrollId, Client client) { - super(client, SearchScrollAction.INSTANCE, scrollId); - } - - @Override - public SearchResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public SearchResponse get(TimeValue timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public SearchResponse get(String timeout) { - throw new IllegalStateException("Not yet implemented"); - } - - @Override - public ListenableActionFuture<SearchResponse> execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public String toString() { - return String.format("ES search scroll request for scroll id '%s'", super.request().scroll()); - } -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/response/ClusterStatsResponse.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/ClusterStatsResponse.java new file mode 100644 index 00000000000..a87acd45d18 --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/ClusterStatsResponse.java @@ -0,0 +1,48 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.response; + +import com.google.gson.JsonObject; +import javax.annotation.concurrent.Immutable; +import org.elasticsearch.cluster.health.ClusterHealthStatus; + +@Immutable +public class ClusterStatsResponse { + + private final ClusterHealthStatus healthStatus; + private final int nodeCount; + + private ClusterStatsResponse(JsonObject clusterStatsJson) { + this.healthStatus = ClusterHealthStatus.fromString(clusterStatsJson.get("status").getAsString()); + this.nodeCount = clusterStatsJson.getAsJsonObject("nodes").getAsJsonObject("count").get("total").getAsInt(); + } + + public static ClusterStatsResponse toClusterStatsResponse(JsonObject jsonObject) { + return new ClusterStatsResponse(jsonObject); + } + + public ClusterHealthStatus getHealthStatus() { + return healthStatus; + } + + public int getNodeCount() { + return nodeCount; + } +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/response/IndexStats.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/IndexStats.java new file mode 100644 index 00000000000..6bbdd8ba11e --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/IndexStats.java @@ -0,0 +1,58 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.response; + +import com.google.gson.JsonObject; +import javax.annotation.concurrent.Immutable; + +@Immutable +public class IndexStats { + private final String name; + private final long docCount; + private final long shardsCount; + private final long storeSizeBytes; + + private IndexStats(String name, JsonObject indexStatsJson) { + this.name = name; + this.docCount = indexStatsJson.getAsJsonObject().getAsJsonObject("primaries").getAsJsonObject("docs").get("count").getAsLong(); + this.shardsCount = indexStatsJson.getAsJsonObject().getAsJsonObject("shards").size(); + this.storeSizeBytes = indexStatsJson.getAsJsonObject().getAsJsonObject("primaries").getAsJsonObject("store").get("size_in_bytes").getAsLong(); + } + + static IndexStats toIndexStats(String name, JsonObject indexStatsJson) { + return new IndexStats(name, indexStatsJson); + } + + public String getName() { + return name; + } + + public long getDocCount() { + return docCount; + } + + public long getShardsCount() { + return shardsCount; + } + + public long getStoreSizeBytes() { + return storeSizeBytes; + } +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/response/IndicesStats.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/IndicesStats.java new file mode 100644 index 00000000000..75928920bfd --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/IndicesStats.java @@ -0,0 +1,65 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.response; + +import com.google.gson.JsonObject; +import javax.annotation.concurrent.Immutable; + +@Immutable +public class IndicesStats { + private static final String MEMORY_SIZE_IN_BYTES_ATTRIBUTE_NAME = "memory_size_in_bytes"; + private final long storeSizeInBytes; + private final long translogSizeInBytes; + private final long requestCacheMemorySizeInBytes; + private final long fieldDataMemorySizeInBytes; + private final long queryCacheMemorySizeInBytes; + + private IndicesStats(JsonObject indicesStatsJson) { + this.storeSizeInBytes = indicesStatsJson.getAsJsonObject("store").get("size_in_bytes").getAsLong(); + this.translogSizeInBytes = indicesStatsJson.getAsJsonObject("translog").get("size_in_bytes").getAsLong(); + this.requestCacheMemorySizeInBytes = indicesStatsJson.getAsJsonObject("request_cache").get(MEMORY_SIZE_IN_BYTES_ATTRIBUTE_NAME).getAsLong(); + this.fieldDataMemorySizeInBytes = indicesStatsJson.getAsJsonObject("fielddata").get(MEMORY_SIZE_IN_BYTES_ATTRIBUTE_NAME).getAsLong(); + this.queryCacheMemorySizeInBytes = indicesStatsJson.getAsJsonObject("query_cache").get(MEMORY_SIZE_IN_BYTES_ATTRIBUTE_NAME).getAsLong(); + } + + public static IndicesStats toIndicesStats(JsonObject jsonObject) { + return new IndicesStats(jsonObject); + } + + public long getStoreSizeInBytes() { + return storeSizeInBytes; + } + + public long getTranslogSizeInBytes() { + return translogSizeInBytes; + } + + public long getRequestCacheMemorySizeInBytes() { + return requestCacheMemorySizeInBytes; + } + + public long getFieldDataMemorySizeInBytes() { + return fieldDataMemorySizeInBytes; + } + + public long getQueryCacheMemorySizeInBytes() { + return queryCacheMemorySizeInBytes; + } +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/response/IndicesStatsResponse.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/IndicesStatsResponse.java new file mode 100644 index 00000000000..0762c395024 --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/IndicesStatsResponse.java @@ -0,0 +1,50 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.response; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import java.util.Collection; +import java.util.Map; +import javax.annotation.concurrent.Immutable; + +@Immutable +public class IndicesStatsResponse { + + private final ImmutableMap<String, IndexStats> indexStatsMap; + + private IndicesStatsResponse(JsonObject indicesStats) { + ImmutableMap.Builder<String, IndexStats> builder = ImmutableMap.builder(); + for (Map.Entry<String, JsonElement> indexStats : indicesStats.getAsJsonObject("indices").entrySet()) { + builder.put(indexStats.getKey(), IndexStats.toIndexStats(indexStats.getKey(), indexStats.getValue().getAsJsonObject())); + } + this.indexStatsMap = builder.build(); + } + + public static IndicesStatsResponse toIndicesStatsResponse(JsonObject jsonObject) { + return new IndicesStatsResponse(jsonObject); + } + + public Collection<IndexStats> getAllIndexStats() { + return indexStatsMap.values(); + } + +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/response/JvmStats.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/JvmStats.java new file mode 100644 index 00000000000..31369d8afb0 --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/JvmStats.java @@ -0,0 +1,64 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.response; + +import com.google.gson.JsonObject; +import javax.annotation.concurrent.Immutable; + +@Immutable +public class JvmStats { + private final long heapUsedPercent; + private final long heapUsedInBytes; + private final long heapMaxInBytes; + private final long nonHeapUsedInBytes; + private final long threadCount; + + private JvmStats(JsonObject jvmStatsJson) { + this.heapUsedPercent = jvmStatsJson.getAsJsonObject("mem").get("heap_used_percent").getAsLong(); + this.heapUsedInBytes = jvmStatsJson.getAsJsonObject("mem").get("heap_used_in_bytes").getAsLong(); + this.heapMaxInBytes = jvmStatsJson.getAsJsonObject("mem").get("heap_max_in_bytes").getAsLong(); + this.nonHeapUsedInBytes = jvmStatsJson.getAsJsonObject("mem").get("non_heap_used_in_bytes").getAsLong(); + this.threadCount = jvmStatsJson.getAsJsonObject("threads").get("count").getAsLong(); + } + + public static JvmStats toJvmStats(JsonObject jvmStatsJson) { + return new JvmStats(jvmStatsJson); + } + + public long getHeapUsedPercent() { + return heapUsedPercent; + } + + public long getHeapUsedInBytes() { + return heapUsedInBytes; + } + + public long getHeapMaxInBytes() { + return heapMaxInBytes; + } + + public long getNonHeapUsedInBytes() { + return nonHeapUsedInBytes; + } + + public long getThreadCount() { + return threadCount; + } +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/response/NodeStats.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/NodeStats.java new file mode 100644 index 00000000000..f7249380e43 --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/NodeStats.java @@ -0,0 +1,115 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.response; + +import com.google.gson.JsonObject; +import javax.annotation.concurrent.Immutable; + +@Immutable +public class NodeStats { + private static final String PROCESS_ATTRIBUTE_NAME = "process"; + private static final String BREAKERS_ATTRIBUTE_NAME = "breakers"; + + private final String name; + private final String host; + + private final long cpuUsage; + private final long openFileDescriptors; + private final long maxFileDescriptors; + private final long diskAvailableBytes; + + private final long fieldDataCircuitBreakerLimit; + private final long fieldDataCircuitBreakerEstimation; + private final long requestCircuitBreakerLimit; + private final long requestCircuitBreakerEstimation; + + private final JvmStats jvmStats; + private final IndicesStats indicesStats; + + private NodeStats(JsonObject nodeStatsJson) { + this.name = nodeStatsJson.get("name").getAsString(); + this.host = nodeStatsJson.get("host").getAsString(); + + this.cpuUsage = nodeStatsJson.getAsJsonObject(PROCESS_ATTRIBUTE_NAME).getAsJsonObject("cpu").get("percent").getAsLong(); + this.openFileDescriptors = nodeStatsJson.getAsJsonObject(PROCESS_ATTRIBUTE_NAME).get("open_file_descriptors").getAsLong(); + this.maxFileDescriptors = nodeStatsJson.getAsJsonObject(PROCESS_ATTRIBUTE_NAME).get("max_file_descriptors").getAsLong(); + this.diskAvailableBytes = nodeStatsJson.getAsJsonObject("fs").getAsJsonObject("total").get("available_in_bytes").getAsLong(); + + this.fieldDataCircuitBreakerLimit = nodeStatsJson.getAsJsonObject(BREAKERS_ATTRIBUTE_NAME).getAsJsonObject("fielddata").get("limit_size_in_bytes").getAsLong(); + this.fieldDataCircuitBreakerEstimation = nodeStatsJson.getAsJsonObject(BREAKERS_ATTRIBUTE_NAME).getAsJsonObject("fielddata").get("estimated_size_in_bytes").getAsLong(); + this.requestCircuitBreakerLimit = nodeStatsJson.getAsJsonObject(BREAKERS_ATTRIBUTE_NAME).getAsJsonObject("request").get("limit_size_in_bytes").getAsLong(); + this.requestCircuitBreakerEstimation = nodeStatsJson.getAsJsonObject(BREAKERS_ATTRIBUTE_NAME).getAsJsonObject("request").get("estimated_size_in_bytes").getAsLong(); + + this.jvmStats = JvmStats.toJvmStats(nodeStatsJson.getAsJsonObject("jvm")); + this.indicesStats = IndicesStats.toIndicesStats(nodeStatsJson.getAsJsonObject("indices")); + } + + public static NodeStats toNodeStats(JsonObject jsonObject) { + return new NodeStats(jsonObject); + } + + public String getName() { + return name; + } + + public String getHost() { + return host; + } + + public long getCpuUsage() { + return cpuUsage; + } + + public long getOpenFileDescriptors() { + return openFileDescriptors; + } + + public long getMaxFileDescriptors() { + return maxFileDescriptors; + } + + public long getDiskAvailableBytes() { + return diskAvailableBytes; + } + + public long getFieldDataCircuitBreakerLimit() { + return fieldDataCircuitBreakerLimit; + } + + public long getFieldDataCircuitBreakerEstimation() { + return fieldDataCircuitBreakerEstimation; + } + + public long getRequestCircuitBreakerLimit() { + return requestCircuitBreakerLimit; + } + + public long getRequestCircuitBreakerEstimation() { + return requestCircuitBreakerEstimation; + } + + public JvmStats getJvmStats() { + return jvmStats; + } + + public IndicesStats getIndicesStats() { + return indicesStats; + } +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/response/NodeStatsResponse.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/NodeStatsResponse.java new file mode 100644 index 00000000000..774459487a2 --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/NodeStatsResponse.java @@ -0,0 +1,51 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.response; + +import com.google.common.collect.ImmutableList; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import java.util.List; +import java.util.Map; +import javax.annotation.concurrent.Immutable; +import org.sonar.core.util.stream.MoreCollectors; + +@Immutable +public class NodeStatsResponse { + + private final ImmutableList<NodeStats> nodeStats; + + private NodeStatsResponse(JsonObject nodeStatsJson) { + this.nodeStats = nodeStatsJson.getAsJsonObject("nodes").entrySet().stream() + .map(Map.Entry::getValue) + .map(JsonElement::getAsJsonObject) + .map(NodeStats::toNodeStats) + .collect(MoreCollectors.toList()); + } + + public static NodeStatsResponse toNodeStatsResponse(JsonObject jsonObject) { + return new NodeStatsResponse(jsonObject); + } + + public List<NodeStats> getNodeStats() { + return this.nodeStats; + } + +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java index 3e78f12b19f..69233bf3c1c 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java @@ -28,7 +28,8 @@ import java.util.Iterator; import java.util.List; import java.util.Set; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.sonar.api.resources.Qualifiers; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; @@ -262,22 +263,22 @@ public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer { } private IndexRequest newIndexRequest(IssueDoc issue) { - return esClient.prepareIndex(TYPE_ISSUE.getMainType()) - .setId(issue.getId()) - .setRouting(issue.getRouting().orElseThrow(() -> new IllegalStateException("IssueDoc should define a routing"))) - .setSource(issue.getFields()) - .request(); + return new IndexRequest(TYPE_ISSUE.getMainType().getIndex().getName(), TYPE_ISSUE.getMainType().getType()) + .id(issue.getId()) + .routing(issue.getRouting().orElseThrow(() -> new IllegalStateException("IssueDoc should define a routing"))) + .source(issue.getFields()); } private void addProjectDeletionToBulkIndexer(BulkIndexer bulkIndexer, String projectUuid) { - SearchRequestBuilder search = esClient.prepareSearch(TYPE_ISSUE.getMainType()) - .setRouting(AuthorizationDoc.idOf(projectUuid)) - .setQuery(boolQuery().must(termQuery(FIELD_ISSUE_PROJECT_UUID, projectUuid))); + SearchRequest search = EsClient.prepareSearch(TYPE_ISSUE.getMainType()) + .routing(AuthorizationDoc.idOf(projectUuid)) + .source(new SearchSourceBuilder().query(boolQuery().must(termQuery(FIELD_ISSUE_PROJECT_UUID, projectUuid)))); + bulkIndexer.addDeletion(search); } private static EsQueueDto createQueueDto(String docId, String docIdType, String projectUuid) { - return EsQueueDto.create(TYPE_ISSUE.format(), docId, docIdType, projectUuid); + return EsQueueDto.create(TYPE_ISSUE.format(), docId, docIdType, AuthorizationDoc.idOf(projectUuid)); } private BulkIndexer createBulkIndexer(Size size, IndexingListener listener) { diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java index 92387e6b754..02f9f2a1c92 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java @@ -27,8 +27,9 @@ import java.util.Map; import java.util.Set; import javax.annotation.Nullable; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; import org.sonar.db.DbClient; @@ -184,8 +185,8 @@ public class ActiveRuleIndexer implements ResilientIndexer { RulesProfileDto profile = dbClient.qualityProfileDao().selectRuleProfile(dbSession, ruleProfileUUid); if (profile == null) { // profile does not exist anymore in db --> related documents must be deleted from index rules/activeRule - SearchRequestBuilder search = esClient.prepareSearch(TYPE_ACTIVE_RULE.getMainType()) - .setQuery(QueryBuilders.boolQuery().must(termQuery(FIELD_ACTIVE_RULE_PROFILE_UUID, ruleProfileUUid))); + SearchRequest search = EsClient.prepareSearch(TYPE_ACTIVE_RULE.getMainType()) + .source(new SearchSourceBuilder().query(QueryBuilders.boolQuery().must(termQuery(FIELD_ACTIVE_RULE_PROFILE_UUID, ruleProfileUUid)))); profileResult = BulkIndexer.delete(esClient, TYPE_ACTIVE_RULE, search); } else { diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndex.java b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndex.java index e30b69aed3c..ead195c2e16 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndex.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndex.java @@ -31,7 +31,7 @@ import java.util.function.Function; import javax.annotation.Nullable; import org.apache.commons.lang.StringUtils; import org.apache.lucene.search.join.ScoreMode; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.BoolQueryBuilder; @@ -46,6 +46,7 @@ import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; @@ -147,38 +148,40 @@ public class RuleIndex { } public SearchIdResult<String> search(RuleQuery query, SearchOptions options) { - SearchRequestBuilder esSearch = client.prepareSearch(TYPE_RULE); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); QueryBuilder qb = buildQuery(query); Map<String, QueryBuilder> filters = buildFilters(query); if (!options.getFacets().isEmpty()) { for (AggregationBuilder aggregation : getFacets(query, options, qb, filters).values()) { - esSearch.addAggregation(aggregation); + sourceBuilder.aggregation(aggregation); } } - setSorting(query, esSearch); - setPagination(options, esSearch); + setSorting(query, sourceBuilder); + setPagination(options, sourceBuilder); BoolQueryBuilder fb = boolQuery(); for (QueryBuilder filterBuilder : filters.values()) { fb.must(filterBuilder); } - esSearch.setQuery(boolQuery().must(qb).filter(fb)); - return new SearchIdResult<>(esSearch.get(), input -> input, system2.getDefaultTimeZone()); + sourceBuilder.query(boolQuery().must(qb).filter(fb)); + + SearchRequest esSearch = EsClient.prepareSearch(TYPE_RULE) + .source(sourceBuilder); + + return new SearchIdResult<>(client.search(esSearch), input -> input, system2.getDefaultTimeZone()); } /** * Return all rule uuids matching the search query, without pagination nor facets */ public Iterator<String> searchAll(RuleQuery query) { - SearchRequestBuilder esSearch = client - .prepareSearch(TYPE_RULE) - .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); - optimizeScrollRequest(esSearch); + optimizeScrollRequest(sourceBuilder); QueryBuilder qb = buildQuery(query); Map<String, QueryBuilder> filters = buildFilters(query); @@ -187,8 +190,13 @@ public class RuleIndex { fb.must(filterBuilder); } - esSearch.setQuery(boolQuery().must(qb).filter(fb)); - SearchResponse response = esSearch.get(); + sourceBuilder.query(boolQuery().must(qb).filter(fb)); + + SearchRequest esSearch = EsClient.prepareSearch(TYPE_RULE) + .scroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)) + .source(sourceBuilder); + + SearchResponse response = client.search(esSearch); return scrollIds(client, response, i -> i); } @@ -561,7 +569,7 @@ public class RuleIndex { return new StickyFacetBuilder(query, filters, null, BucketOrder.compound(BucketOrder.count(false), BucketOrder.key(true))); } - private static void setSorting(RuleQuery query, SearchRequestBuilder esSearch) { + private static void setSorting(RuleQuery query, SearchSourceBuilder esSearch) { /* integrate Query Sort */ String queryText = query.getQueryText(); if (query.getSortField() != null) { @@ -571,13 +579,13 @@ public class RuleIndex { } else { sort.order(SortOrder.DESC); } - esSearch.addSort(sort); + esSearch.sort(sort); } else if (StringUtils.isNotEmpty(queryText)) { - esSearch.addSort(SortBuilders.scoreSort()); + esSearch.sort(SortBuilders.scoreSort()); } else { - esSearch.addSort(appendSortSuffixIfNeeded(FIELD_RULE_UPDATED_AT), SortOrder.DESC); + esSearch.sort(appendSortSuffixIfNeeded(FIELD_RULE_UPDATED_AT), SortOrder.DESC); // deterministic sort when exactly the same updated_at (same millisecond) - esSearch.addSort(appendSortSuffixIfNeeded(FIELD_RULE_KEY), SortOrder.ASC); + esSearch.sort(appendSortSuffixIfNeeded(FIELD_RULE_KEY), SortOrder.ASC); } } @@ -588,9 +596,9 @@ public class RuleIndex { : ""); } - private static void setPagination(SearchOptions options, SearchRequestBuilder esSearch) { - esSearch.setFrom(options.getOffset()); - esSearch.setSize(options.getLimit()); + private static void setPagination(SearchOptions options, SearchSourceBuilder esSearch) { + esSearch.from(options.getOffset()); + esSearch.size(options.getLimit()); } public List<String> listTags(@Nullable String query, int size) { @@ -611,13 +619,13 @@ public class RuleIndex { .map(s -> new IncludeExclude(s, null)) .ifPresent(termsAggregation::includeExclude); - SearchRequestBuilder request = client - .prepareSearch(TYPE_RULE.getMainType()) - .setQuery(matchAllQuery()) - .setSize(0) - .addAggregation(termsAggregation); + SearchRequest request = EsClient.prepareSearch(TYPE_RULE.getMainType()) + .source(new SearchSourceBuilder() + .query(matchAllQuery()) + .size(0) + .aggregation(termsAggregation)); - SearchResponse esResponse = request.get(); + SearchResponse esResponse = client.search(request); return EsUtils.termsKeys(esResponse.getAggregations().get(AGGREGATION_NAME_FOR_TAGS)); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/user/index/UserIndex.java b/server/sonar-server-common/src/main/java/org/sonar/server/user/index/UserIndex.java index 5dbd1803e0a..75d142cd6ee 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/user/index/UserIndex.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/user/index/UserIndex.java @@ -23,12 +23,14 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.Operator; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortOrder; import org.sonar.api.ce.ComputeEngineSide; import org.sonar.api.server.ServerSide; @@ -69,15 +71,17 @@ public class UserIndex { public List<UserDoc> getAtMostThreeActiveUsersForScmAccount(String scmAccount) { List<UserDoc> result = new ArrayList<>(); if (!StringUtils.isEmpty(scmAccount)) { - SearchRequestBuilder request = esClient.prepareSearch(UserIndexDefinition.TYPE_USER) - .setQuery(boolQuery().must(matchAllQuery()).filter( - boolQuery() - .must(termQuery(FIELD_ACTIVE, true)) - .should(termQuery(FIELD_LOGIN, scmAccount)) - .should(matchQuery(SORTABLE_ANALYZER.subField(FIELD_EMAIL), scmAccount)) - .should(matchQuery(SORTABLE_ANALYZER.subField(FIELD_SCM_ACCOUNTS), scmAccount)))) - .setSize(3); - for (SearchHit hit : request.get().getHits().getHits()) { + SearchResponse response = esClient.search(EsClient.prepareSearch(UserIndexDefinition.TYPE_USER) + .source(new SearchSourceBuilder() + .query(boolQuery().must(matchAllQuery()).filter( + boolQuery() + .must(termQuery(FIELD_ACTIVE, true)) + .should(termQuery(FIELD_LOGIN, scmAccount)) + .should(matchQuery(SORTABLE_ANALYZER.subField(FIELD_EMAIL), scmAccount)) + .should(matchQuery(SORTABLE_ANALYZER.subField(FIELD_SCM_ACCOUNTS), scmAccount)) + .minimumShouldMatch(1))) + .size(3))); + for (SearchHit hit : response.getHits().getHits()) { result.add(new UserDoc(hit.getSourceAsMap())); } } @@ -85,10 +89,10 @@ public class UserIndex { } public SearchResult<UserDoc> search(UserQuery userQuery, SearchOptions options) { - SearchRequestBuilder request = esClient.prepareSearch(UserIndexDefinition.TYPE_USER) - .setSize(options.getLimit()) - .setFrom(options.getOffset()) - .addSort(FIELD_NAME, SortOrder.ASC); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .size(options.getLimit()) + .from(options.getOffset()) + .sort(FIELD_NAME, SortOrder.ASC); BoolQueryBuilder filter = boolQuery().must(termQuery(FIELD_ACTIVE, true)); userQuery.getOrganizationUuid() @@ -109,9 +113,9 @@ public class UserIndex { .operator(Operator.AND); } - request.setQuery(boolQuery().must(esQuery).filter(filter)); - - return new SearchResult<>(request.get(), UserDoc::new, system2.getDefaultTimeZone()); + SearchRequest request = EsClient.prepareSearch(UserIndexDefinition.TYPE_USER) + .source(searchSourceBuilder.query(boolQuery().must(esQuery).filter(filter))); + return new SearchResult<>(esClient.search(request), UserDoc::new, system2.getDefaultTimeZone()); } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndex.java b/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndex.java index 83eff7fe0a9..3ed04ff5f49 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndex.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndex.java @@ -20,10 +20,13 @@ package org.sonar.server.view.index; import java.util.List; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.ClearScrollRequest; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortOrder; import org.sonar.api.ce.ComputeEngineSide; import org.sonar.api.server.ServerSide; @@ -45,14 +48,15 @@ public class ViewIndex { } public List<String> findAllViewUuids() { - SearchRequestBuilder esSearch = esClient.prepareSearch(ViewIndexDefinition.TYPE_VIEW) - .addSort("_doc", SortOrder.ASC) - .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)) - .setFetchSource(false) - .setSize(100) - .setQuery(matchAllQuery()); + SearchRequest esSearch = EsClient.prepareSearch(ViewIndexDefinition.TYPE_VIEW) + .source(new SearchSourceBuilder() + .sort("_doc", SortOrder.ASC) + .fetchSource(false) + .size(100) + .query(matchAllQuery())) + .scroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)); - SearchResponse response = esSearch.get(); + SearchResponse response = esClient.search(esSearch); List<String> result = newArrayList(); while (true) { List<SearchHit> hits = newArrayList(response.getHits()); @@ -60,12 +64,12 @@ public class ViewIndex { result.add(hit.getId()); } String scrollId = response.getScrollId(); - response = esClient.prepareSearchScroll(scrollId) - .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)) - .get(); + response = esClient.scroll(new SearchScrollRequest().scrollId(scrollId).scroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES))); // Break condition: No hits are returned if (response.getHits().getHits().length == 0) { - esClient.nativeClient().prepareClearScroll().addScrollId(scrollId).get(); + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); + clearScrollRequest.addScrollId(scrollId); + esClient.clearScroll(clearScrollRequest); break; } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndexer.java index c1a5be8d107..10b119153ac 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndexer.java @@ -25,6 +25,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; import org.elasticsearch.action.index.IndexRequest; import org.sonar.core.util.stream.MoreCollectors; import org.sonar.db.DbClient; @@ -128,9 +129,7 @@ public class ViewIndexer implements ResilientIndexer { private void clearLookupCache(String viewUuid) { try { - esClient.prepareClearCache() - .setQueryCache(true) - .get(); + esClient.clearCache(new ClearIndicesCacheRequest().queryCache(true)); } catch (Exception e) { throw new IllegalStateException(String.format("Unable to clear lookup cache of view '%s'", viewUuid), e); } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/component/index/ComponentIndexerTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/component/index/ComponentIndexerTest.java index 1589043bd0b..af3d714d8ce 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/component/index/ComponentIndexerTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/component/index/ComponentIndexerTest.java @@ -22,6 +22,7 @@ package org.sonar.server.component.index; import java.util.Arrays; import java.util.Collection; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.Rule; import org.junit.Test; import org.sonar.api.utils.System2; @@ -32,6 +33,7 @@ import org.sonar.db.component.ComponentDto; import org.sonar.db.component.ComponentUpdateDto; import org.sonar.db.es.EsQueueDto; import org.sonar.db.organization.OrganizationDto; +import org.sonar.server.es.EsClient; import org.sonar.server.es.EsTester; import org.sonar.server.es.IndexingResult; import org.sonar.server.es.ProjectIndexer; @@ -257,9 +259,9 @@ public class ComponentIndexerTest { private void assertThatComponentHasName(ComponentDto component, String expectedName) { SearchHit[] hits = es.client() - .prepareSearch(TYPE_COMPONENT.getMainType()) - .setQuery(matchQuery(SORTABLE_ANALYZER.subField(FIELD_NAME), expectedName)) - .get() + .search(EsClient.prepareSearch(TYPE_COMPONENT.getMainType()) + .source(new SearchSourceBuilder() + .query(matchQuery(SORTABLE_ANALYZER.subField(FIELD_NAME), expectedName)))) .getHits() .getHits(); assertThat(hits) diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/BulkIndexerTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/BulkIndexerTest.java index edaeb253d73..52a0ef0a73b 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/BulkIndexerTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/BulkIndexerTest.java @@ -20,13 +20,17 @@ package org.sonar.server.es; import com.google.common.collect.ImmutableMap; +import java.io.IOException; import java.util.ArrayList; import java.util.List; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.Rule; import org.junit.Test; import org.sonar.api.impl.utils.TestSystem2; @@ -112,8 +116,8 @@ public class BulkIndexerTest { es.putDocuments(TYPE_FAKE, docs); assertThat(count()).isEqualTo(max); - SearchRequestBuilder req = es.client().prepareSearch(TYPE_FAKE) - .setQuery(QueryBuilders.rangeQuery(FakeIndexDefinition.INT_FIELD).gte(removeFrom)); + SearchRequest req = EsClient.prepareSearch(TYPE_FAKE) + .source(new SearchSourceBuilder().query(QueryBuilders.rangeQuery(FakeIndexDefinition.INT_FIELD).gte(removeFrom))); BulkIndexer.delete(es.client(), TYPE_FAKE, req); assertThat(count()).isEqualTo(removeFrom); @@ -197,9 +201,13 @@ public class BulkIndexerTest { } private int replicas() { - GetSettingsResponse settingsResp = es.client().nativeClient().admin().indices() - .prepareGetSettings(INDEX).get(); - return Integer.parseInt(settingsResp.getSetting(INDEX, IndexMetaData.SETTING_NUMBER_OF_REPLICAS)); + try { + GetSettingsResponse settingsResp = es.client().nativeClient().indices() + .getSettings(new GetSettingsRequest().indices(INDEX), RequestOptions.DEFAULT); + return Integer.parseInt(settingsResp.getSetting(INDEX, IndexMetadata.SETTING_NUMBER_OF_REPLICAS)); + } catch (IOException e) { + throw new IllegalStateException("Could not get index settings", e); + } } private IndexRequest newIndexRequest(int intField) { diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientProviderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientProviderTest.java index 7a8b5edbb7e..3ec60eb6237 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientProviderTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientProviderTest.java @@ -21,8 +21,8 @@ package org.sonar.server.es; import java.net.InetAddress; import org.assertj.core.api.Condition; -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.client.Node; +import org.elasticsearch.client.RestHighLevelClient; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -39,6 +39,7 @@ import static org.sonar.process.ProcessProperties.Property.CLUSTER_NODE_TYPE; import static org.sonar.process.ProcessProperties.Property.CLUSTER_SEARCH_HOSTS; import static org.sonar.process.ProcessProperties.Property.SEARCH_HOST; import static org.sonar.process.ProcessProperties.Property.SEARCH_PORT; +import static org.sonar.process.ProcessProperties.Property.SEARCH_TRANSPORT_PORT; public class EsClientProviderTest { @@ -50,29 +51,31 @@ public class EsClientProviderTest { private MapSettings settings = new MapSettings(); private EsClientProvider underTest = new EsClientProvider(); - private String localhost; + private String localhostHostname; @Before public void setUp() throws Exception { // mandatory property settings.setProperty(CLUSTER_NAME.getKey(), "the_cluster_name"); - localhost = InetAddress.getLocalHost().getHostAddress(); + localhostHostname = InetAddress.getLocalHost().getHostName(); } @Test public void connection_to_local_es_when_cluster_mode_is_disabled() { settings.setProperty(CLUSTER_ENABLED.getKey(), false); - settings.setProperty(SEARCH_HOST.getKey(), localhost); - settings.setProperty(SEARCH_PORT.getKey(), 8080); + settings.setProperty(SEARCH_HOST.getKey(), localhostHostname); + settings.setProperty(SEARCH_PORT.getKey(), 9000); + settings.setProperty(SEARCH_TRANSPORT_PORT.getKey(), 8080); EsClient client = underTest.provide(settings.asConfig()); - TransportClient transportClient = (TransportClient) client.nativeClient(); - assertThat(transportClient.transportAddresses()).hasSize(1); - TransportAddress address = transportClient.transportAddresses().get(0); - assertThat(address.getAddress()).isEqualTo(localhost); - assertThat(address.getPort()).isEqualTo(8080); - assertThat(logTester.logs(LoggerLevel.INFO)).has(new Condition<>(s -> s.contains("Connected to local Elasticsearch: [" + localhost + ":8080]"), "")); + RestHighLevelClient nativeClient = client.nativeClient(); + assertThat(nativeClient.getLowLevelClient().getNodes()).hasSize(1); + Node node = nativeClient.getLowLevelClient().getNodes().get(0); + assertThat(node.getHost().getAddress().getHostName()).isEqualTo(localhostHostname); + assertThat(node.getHost().getPort()).isEqualTo(9000); + + assertThat(logTester.logs(LoggerLevel.INFO)).has(new Condition<>(s -> s.contains("Connected to local Elasticsearch: [http://" + localhostHostname + ":9000]"), "")); // keep in cache assertThat(underTest.provide(settings.asConfig())).isSameAs(client); @@ -82,37 +85,41 @@ public class EsClientProviderTest { public void connection_to_remote_es_nodes_when_cluster_mode_is_enabled_and_local_es_is_disabled() { settings.setProperty(CLUSTER_ENABLED.getKey(), true); settings.setProperty(CLUSTER_NODE_TYPE.getKey(), "application"); - settings.setProperty(CLUSTER_SEARCH_HOSTS.getKey(), format("%s:8080,%s:8081", localhost, localhost)); + settings.setProperty(CLUSTER_SEARCH_HOSTS.getKey(), format("%s:8080,%s:8081", localhostHostname, localhostHostname)); EsClient client = underTest.provide(settings.asConfig()); - TransportClient transportClient = (TransportClient) client.nativeClient(); - assertThat(transportClient.transportAddresses()).hasSize(2); - TransportAddress address = transportClient.transportAddresses().get(0); - assertThat(address.getAddress()).isEqualTo(localhost); - assertThat(address.getPort()).isEqualTo(8080); - address = transportClient.transportAddresses().get(1); - assertThat(address.getAddress()).isEqualTo(localhost); - assertThat(address.getPort()).isEqualTo(8081); - assertThat(logTester.logs(LoggerLevel.INFO)).has(new Condition<>(s -> s.contains("Connected to remote Elasticsearch: [" + localhost + ":8080, " + localhost + ":8081]"), "")); + RestHighLevelClient nativeClient = client.nativeClient(); + assertThat(nativeClient.getLowLevelClient().getNodes()).hasSize(2); + + Node node = nativeClient.getLowLevelClient().getNodes().get(0); + assertThat(node.getHost().getAddress().getHostName()).isEqualTo(localhostHostname); + assertThat(node.getHost().getPort()).isEqualTo(8080); + + node = nativeClient.getLowLevelClient().getNodes().get(1); + assertThat(node.getHost().getAddress().getHostName()).isEqualTo(localhostHostname); + assertThat(node.getHost().getPort()).isEqualTo(8081); + + assertThat(logTester.logs(LoggerLevel.INFO)) + .has(new Condition<>(s -> s.contains("Connected to remote Elasticsearch: [http://" + localhostHostname + ":8080, http://" + localhostHostname + ":8081]"), "")); // keep in cache assertThat(underTest.provide(settings.asConfig())).isSameAs(client); } @Test - public void es_client_provider_must_throw_ISE_when_incorrect_port_is_used_when_search_disabled() { + public void es_client_provider_must_throw_IAE_when_incorrect_port_is_used_when_search_disabled() { settings.setProperty(CLUSTER_ENABLED.getKey(), true); settings.setProperty(CLUSTER_NODE_TYPE.getKey(), "application"); - settings.setProperty(CLUSTER_SEARCH_HOSTS.getKey(), format("%s:100000,%s:8081", localhost, localhost)); + settings.setProperty(CLUSTER_SEARCH_HOSTS.getKey(), format("%s:100000,%s:8081", localhostHostname, localhostHostname)); expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage(format("Port number out of range: %s:100000", localhost)); + expectedException.expectMessage(format("Port number out of range: %s:100000", localhostHostname)); underTest.provide(settings.asConfig()); } @Test - public void es_client_provider_must_throw_ISE_when_incorrect_port_is_used() { + public void es_client_provider_must_throw_IAE_when_incorrect_port_is_used() { settings.setProperty(CLUSTER_ENABLED.getKey(), true); settings.setProperty(CLUSTER_NODE_TYPE.getKey(), "search"); settings.setProperty(SEARCH_HOST.getKey(), "localhost"); @@ -128,18 +135,22 @@ public class EsClientProviderTest { public void es_client_provider_must_add_default_port_when_not_specified() { settings.setProperty(CLUSTER_ENABLED.getKey(), true); settings.setProperty(CLUSTER_NODE_TYPE.getKey(), "application"); - settings.setProperty(CLUSTER_SEARCH_HOSTS.getKey(), format("%s,%s:8081", localhost, localhost)); + settings.setProperty(CLUSTER_SEARCH_HOSTS.getKey(), format("%s,%s:8081", localhostHostname, localhostHostname)); EsClient client = underTest.provide(settings.asConfig()); - TransportClient transportClient = (TransportClient) client.nativeClient(); - assertThat(transportClient.transportAddresses()).hasSize(2); - TransportAddress address = transportClient.transportAddresses().get(0); - assertThat(address.getAddress()).isEqualTo(localhost); - assertThat(address.getPort()).isEqualTo(9001); - address = transportClient.transportAddresses().get(1); - assertThat(address.getAddress()).isEqualTo(localhost); - assertThat(address.getPort()).isEqualTo(8081); - assertThat(logTester.logs(LoggerLevel.INFO)).has(new Condition<>(s -> s.contains("Connected to remote Elasticsearch: [" + localhost + ":9001, " + localhost + ":8081]"), "")); + RestHighLevelClient nativeClient = client.nativeClient(); + assertThat(nativeClient.getLowLevelClient().getNodes()).hasSize(2); + + Node node = nativeClient.getLowLevelClient().getNodes().get(0); + assertThat(node.getHost().getAddress().getHostName()).isEqualTo(localhostHostname); + assertThat(node.getHost().getPort()).isEqualTo(9001); + + node = nativeClient.getLowLevelClient().getNodes().get(1); + assertThat(node.getHost().getAddress().getHostName()).isEqualTo(localhostHostname); + assertThat(node.getHost().getPort()).isEqualTo(8081); + + assertThat(logTester.logs(LoggerLevel.INFO)) + .has(new Condition<>(s -> s.contains("Connected to remote Elasticsearch: [http://" + localhostHostname + ":9001, http://" + localhostHostname + ":8081]"), "")); // keep in cache assertThat(underTest.provide(settings.asConfig())).isSameAs(client); diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientStopperTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientStopperTest.java index 664637caf7b..4c786cb5e64 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientStopperTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientStopperTest.java @@ -23,8 +23,7 @@ import org.junit.Test; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; - +import static org.mockito.Mockito.verifyNoMoreInteractions; public class EsClientStopperTest { @@ -34,7 +33,7 @@ public class EsClientStopperTest { @Test public void stop_client() { underTest.start(); - verifyZeroInteractions(client); + verifyNoMoreInteractions(client); underTest.stop(); verify(client).close(); diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientTest.java index e485491a86d..f08e0d3b403 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientTest.java @@ -19,51 +19,233 @@ */ package org.sonar.server.es; -import org.junit.Rule; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.Objects; +import org.apache.http.HttpEntity; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; import org.junit.Test; -import org.sonar.server.es.newindex.FakeIndexDefinition; -import org.sonar.server.es.request.ProxyClusterHealthRequestBuilder; -import org.sonar.server.es.request.ProxyClusterStateRequestBuilder; -import org.sonar.server.es.request.ProxyClusterStatsRequestBuilder; -import org.sonar.server.es.request.ProxyCreateIndexRequestBuilder; -import org.sonar.server.es.request.ProxyDeleteRequestBuilder; -import org.sonar.server.es.request.ProxyGetRequestBuilder; -import org.sonar.server.es.request.ProxyIndicesExistsRequestBuilder; -import org.sonar.server.es.request.ProxyIndicesStatsRequestBuilder; -import org.sonar.server.es.request.ProxyNodesStatsRequestBuilder; -import org.sonar.server.es.request.ProxyPutMappingRequestBuilder; -import org.sonar.server.es.request.ProxyRefreshRequestBuilder; -import org.sonar.server.es.request.ProxySearchRequestBuilder; -import org.sonar.server.es.request.ProxySearchScrollRequestBuilder; +import org.mockito.ArgumentMatcher; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class EsClientTest { + private static final String EXAMPLE_CLUSTER_STATS_JSON = "{" + + " \"status\": \"yellow\"," + + " \"nodes\": {" + + " \"count\": {" + + " \"total\": 3" + + " }" + + " }" + + "}"; - @Rule - public EsTester es = EsTester.createCustom(new FakeIndexDefinition()); + private static final String EXAMPLE_INDICES_STATS_JSON = "{" + + " \"indices\": {" + + " \"index-1\": {" + + " \"primaries\": {" + + " \"docs\": {" + + " \"count\": 1234" + + " }," + + " \"store\": {" + + " \"size_in_bytes\": 56789" + + " }" + + " }," + + " \"shards\": {" + + " \"shard-1\": {}," + + " \"shard-2\": {}" + + " }" + + " }," + + " \"index-2\": {" + + " \"primaries\": {" + + " \"docs\": {" + + " \"count\": 42" + + " }," + + " \"store\": {" + + " \"size_in_bytes\": 123" + + " }" + + " }," + + " \"shards\": {" + + " \"shard-1\": {}," + + " \"shard-2\": {}" + + " }" + + " }" + + " }" + + "}"; + + private final static String EXAMPLE_NODE_STATS_JSON = "{" + + " \"nodes\": {" + + " \"YnKPZcbGRamRQGxjErLWoQ\": {" + + " \"name\": \"sonarqube\"," + + " \"host\": \"127.0.0.1\"," + + " \"indices\": {" + + " \"docs\": {" + + " \"count\": 13557" + + " }," + + " \"store\": {" + + " \"size_in_bytes\": 8670970" + + " }," + + " \"query_cache\": {" + + " \"memory_size_in_bytes\": 0" + + " }," + + " \"fielddata\": {" + + " \"memory_size_in_bytes\": 4880" + + " }," + + " \"translog\": {" + + " \"size_in_bytes\": 8274137" + + " }," + + " \"request_cache\": {" + + " \"memory_size_in_bytes\": 0" + + " }" + + " }," + + " \"process\": {" + + " \"open_file_descriptors\": 296," + + " \"max_file_descriptors\": 10240," + + " \"cpu\": {" + + " \"percent\": 7" + + " }" + + " }," + + " \"jvm\": {" + + " \"mem\": {" + + " \"heap_used_in_bytes\": 158487160," + + " \"heap_used_percent\": 30," + + " \"heap_max_in_bytes\": 518979584," + + " \"non_heap_used_in_bytes\": 109066592" + + " }," + + " \"threads\": {" + + " \"count\": 70" + + " }" + + " }," + + " \"fs\": {" + + " \"total\": {" + + " \"total_in_bytes\": 250685575168," + + " \"free_in_bytes\": 142843138048," + + " \"available_in_bytes\": 136144027648" + + " }" + + " }," + + " \"breakers\": {" + + " \"request\": {" + + " \"limit_size_in_bytes\": 311387750," + + " \"estimated_size_in_bytes\": 0" + + " }," + + " \"fielddata\": {" + + " \"limit_size_in_bytes\": 207591833," + + " \"estimated_size_in_bytes\": 4880" + + " }" + + " }" + + " }" + + " }" + + "}"; + + RestClient restClient = mock(RestClient.class); + RestHighLevelClient client = new EsClient.MinimalRestHighLevelClient(restClient); + + EsClient underTest = new EsClient(client); @Test - public void proxify_requests() { - Index fakesIndex = Index.simple("fakes"); - IndexType.IndexMainType fakeMainType = IndexType.main(fakesIndex, "fake"); - - EsClient underTest = es.client(); - assertThat(underTest.nativeClient()).isNotNull(); - assertThat(underTest.prepareClusterStats()).isInstanceOf(ProxyClusterStatsRequestBuilder.class); - assertThat(underTest.prepareCreate(fakesIndex)).isInstanceOf(ProxyCreateIndexRequestBuilder.class); - assertThat(underTest.prepareDelete(fakeMainType, "my_id")).isInstanceOf(ProxyDeleteRequestBuilder.class); - assertThat(underTest.prepareIndicesExist(fakesIndex)).isInstanceOf(ProxyIndicesExistsRequestBuilder.class); - assertThat(underTest.prepareGet(fakeMainType, "1")).isInstanceOf(ProxyGetRequestBuilder.class); - assertThat(underTest.prepareHealth()).isInstanceOf(ProxyClusterHealthRequestBuilder.class); - assertThat(underTest.prepareNodesStats()).isInstanceOf(ProxyNodesStatsRequestBuilder.class); - assertThat(underTest.preparePutMapping(fakesIndex)).isInstanceOf(ProxyPutMappingRequestBuilder.class); - assertThat(underTest.prepareRefresh(fakesIndex)).isInstanceOf(ProxyRefreshRequestBuilder.class); - assertThat(underTest.prepareSearch(fakesIndex)).isInstanceOf(ProxySearchRequestBuilder.class); - assertThat(underTest.prepareSearchScroll("1234")).isInstanceOf(ProxySearchScrollRequestBuilder.class); - assertThat(underTest.prepareState()).isInstanceOf(ProxyClusterStateRequestBuilder.class); - assertThat(underTest.prepareStats(fakesIndex)).isInstanceOf(ProxyIndicesStatsRequestBuilder.class); + public void should_close_client() throws IOException { + underTest.close(); + verify(restClient).close(); + } + @Test(expected = ElasticsearchException.class) + public void should_rethrow_ex_when_close_client_throws() throws IOException { + doThrow(IOException.class).when(restClient).close(); underTest.close(); } + + @Test + public void should_call_node_stats_api() throws Exception { + HttpEntity entity = mock(HttpEntity.class); + when(entity.getContent()).thenReturn(new ByteArrayInputStream(EXAMPLE_NODE_STATS_JSON.getBytes())); + Response response = mock(Response.class); + when(response.getEntity()).thenReturn(entity); + when(restClient.performRequest(argThat(new RawRequestMatcher( + "GET", + "/_nodes/stats/fs,process,jvm,indices,breaker")))) + .thenReturn(response); + + assertThat(underTest.nodesStats()).isNotNull(); + } + + @Test(expected = ElasticsearchException.class) + public void should_rethrow_ex_on_node_stat_fail() throws Exception { + when(restClient.performRequest(argThat(new RawRequestMatcher( + "GET", + "/_nodes/stats/fs,process,jvm,indices,breaker")))) + .thenThrow(IOException.class); + underTest.nodesStats(); + } + + @Test + public void should_call_indices_stat_api() throws Exception { + HttpEntity entity = mock(HttpEntity.class); + when(entity.getContent()).thenReturn(new ByteArrayInputStream(EXAMPLE_INDICES_STATS_JSON.getBytes())); + Response response = mock(Response.class); + when(response.getEntity()).thenReturn(entity); + when(restClient.performRequest(argThat(new RawRequestMatcher( + "GET", + "/_stats")))) + .thenReturn(response); + + assertThat(underTest.indicesStats()).isNotNull(); + } + + @Test(expected = ElasticsearchException.class) + public void should_rethrow_ex_on_indices_stat_fail() throws Exception { + when(restClient.performRequest(argThat(new RawRequestMatcher( + "GET", + "/_stats")))) + .thenThrow(IOException.class); + underTest.indicesStats(); + } + + @Test + public void should_call_cluster_stat_api() throws Exception { + HttpEntity entity = mock(HttpEntity.class); + when(entity.getContent()).thenReturn(new ByteArrayInputStream(EXAMPLE_CLUSTER_STATS_JSON.getBytes())); + + Response response = mock(Response.class); + when(response.getEntity()).thenReturn(entity); + when(restClient.performRequest(argThat(new RawRequestMatcher( + "GET", + "/_cluster/stats")))) + .thenReturn(response); + + assertThat(underTest.clusterStats()).isNotNull(); + } + + @Test(expected = ElasticsearchException.class) + public void should_rethrow_ex_on_cluster_stat_fail() throws Exception { + when(restClient.performRequest(argThat(new RawRequestMatcher( + "GET", + "/_cluster/stats")))) + .thenThrow(IOException.class); + underTest.clusterStats(); + } + + static class RawRequestMatcher implements ArgumentMatcher<Request> { + String endpoint; + String method; + + RawRequestMatcher(String method, String endpoint) { + Objects.requireNonNull(endpoint); + Objects.requireNonNull(method); + this.endpoint = endpoint; + this.method = method; + } + + @Override + public boolean matches(Request request) { + return endpoint.equals(request.getEndpoint()) && method.equals(request.getMethod()); + } + } + } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsRequestDetailsTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsRequestDetailsTest.java new file mode 100644 index 00000000000..fe2436cea8e --- /dev/null +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsRequestDetailsTest.java @@ -0,0 +1,154 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es; + +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchScrollRequest; +import org.elasticsearch.client.Requests; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.client.indices.GetIndexRequest; +import org.elasticsearch.common.unit.TimeValue; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class EsRequestDetailsTest { + + @Test + public void should_format_SearchRequest() { + SearchRequest searchRequest = Requests.searchRequest("index") + .types("type"); + assertThat(EsRequestDetails.computeDetailsAsString(searchRequest)) + .isEqualTo("ES search request 'SearchRequest{searchType=QUERY_THEN_FETCH, indices=[index]," + + " indicesOptions=IndicesOptions[ignore_unavailable=false, allow_no_indices=true," + + " expand_wildcards_open=true, expand_wildcards_closed=false, expand_wildcards_hidden=false," + + " allow_aliases_to_multiple_indices=true, forbid_closed_indices=true, ignore_aliases=false," + + " ignore_throttled=true], types=[type], routing='null', preference='null', requestCache=null," + + " scroll=null, maxConcurrentShardRequests=0, batchedReduceSize=512, preFilterShardSize=null," + + " allowPartialSearchResults=null, localClusterAlias=null, getOrCreateAbsoluteStartMillis=-1," + + " ccsMinimizeRoundtrips=true, source={}}' on indices '[index]' on types '[type]'"); + } + + @Test + public void should_format_search_SearchScrollRequest() { + SearchScrollRequest scrollRequest = Requests.searchScrollRequest("scroll-id") + .scroll(TimeValue.ZERO); + assertThat(EsRequestDetails.computeDetailsAsString(scrollRequest)) + .isEqualTo("ES search scroll request for scroll id 'Scroll{keepAlive=0s}'"); + } + + @Test + public void should_format_DeleteRequest() { + DeleteRequest deleteRequest = new DeleteRequest() + .index("some-index") + .type("some-type") + .id("some-id"); + assertThat(EsRequestDetails.computeDetailsAsString(deleteRequest)) + .isEqualTo("ES delete request of doc some-id in index some-index/some-type"); + } + + @Test + public void should_format_RefreshRequest() { + RefreshRequest deleteRequest = new RefreshRequest() + .indices("index-1", "index-2"); + assertThat(EsRequestDetails.computeDetailsAsString(deleteRequest)) + .isEqualTo("ES refresh request on indices 'index-1,index-2'"); + } + + @Test + public void should_format_ClearIndicesCacheRequest() { + ClearIndicesCacheRequest clearIndicesCacheRequest = new ClearIndicesCacheRequest() + .indices("index-1") + .fields("field-1") + .queryCache(true) + .fieldDataCache(true) + .requestCache(true); + assertThat(EsRequestDetails.computeDetailsAsString(clearIndicesCacheRequest)) + .isEqualTo("ES clear cache request on indices 'index-1' on fields 'field-1' with filter cache with field data cache with request cache"); + } + + @Test + public void should_format_IndexRequest() { + IndexRequest indexRequest = new IndexRequest() + .index("index-1") + .type("type-1") + .id("id-1"); + + assertThat(EsRequestDetails.computeDetailsAsString(indexRequest)) + .isEqualTo("ES index request for key 'id-1' on index 'index-1' on type 'type-1'"); + } + + @Test + public void should_format_GetRequest() { + GetRequest request = new GetRequest() + .index("index-1") + .type("type-1") + .id("id-1"); + + assertThat(EsRequestDetails.computeDetailsAsString(request)) + .isEqualTo("ES get request for key 'id-1' on index 'index-1' on type 'type-1'"); + } + + @Test + public void should_format_GetIndexRequest() { + GetIndexRequest request = new GetIndexRequest("index-1", "index-2"); + + assertThat(EsRequestDetails.computeDetailsAsString(request)) + .isEqualTo("ES indices exists request on indices 'index-1,index-2'"); + } + + @Test + public void should_format_CreateIndexRequest() { + CreateIndexRequest request = new CreateIndexRequest("index-1"); + + assertThat(EsRequestDetails.computeDetailsAsString(request)) + .isEqualTo("ES create index 'index-1'"); + } + + @Test + public void should_format_PutMappingRequest() { + PutMappingRequest request = new PutMappingRequest("index-1") + .type("type-1"); + + assertThat(EsRequestDetails.computeDetailsAsString(request)) + .isEqualTo("ES put mapping request on indices 'index-1' on type 'type-1'"); + } + + @Test + public void should_format_ClusterHealthRequest() { + ClusterHealthRequest request = new ClusterHealthRequest("index-1"); + + assertThat(EsRequestDetails.computeDetailsAsString(request)) + .isEqualTo("ES cluster health request on indices 'index-1'"); + } + + @Test + public void should_format_IndicesStats() { + assertThat(EsRequestDetails.computeDetailsAsString("index-1", "index-2")) + .isEqualTo("ES indices stats request on indices 'index-1,index-2'"); + } +} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsUtilsTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsUtilsTest.java index 3b883bdc172..9a068062d83 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsUtilsTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsUtilsTest.java @@ -21,6 +21,7 @@ package org.sonar.server.es; import java.util.Date; import java.util.List; +import org.apache.lucene.search.TotalHits; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.junit.Test; @@ -34,14 +35,14 @@ public class EsUtilsTest { @Test public void convertToDocs_empty() { - SearchHits hits = new SearchHits(new SearchHit[] {}, 0, 0); + SearchHits hits = new SearchHits(new SearchHit[] {}, new TotalHits(0, TotalHits.Relation.EQUAL_TO), 0); List<BaseDoc> docs = EsUtils.convertToDocs(hits, IssueDoc::new); assertThat(docs).isEmpty(); } @Test public void convertToDocs() { - SearchHits hits = new SearchHits(new SearchHit[] {new SearchHit(16)}, 1, 1); + SearchHits hits = new SearchHits(new SearchHit[] {new SearchHit(16)}, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1); List<BaseDoc> docs = EsUtils.convertToDocs(hits, IssueDoc::new); assertThat(docs).hasSize(1); } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/NewIndexTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/NewIndexTest.java index df7601cf480..2a81194dd77 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/NewIndexTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/NewIndexTest.java @@ -24,7 +24,7 @@ import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.DataProviderRunner; import com.tngtech.java.junit.dataprovider.UseDataProvider; import java.util.Map; -import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.junit.Rule; import org.junit.Test; @@ -229,8 +229,8 @@ public class NewIndexTest { public void default_shards_and_replicas(Index index) { NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); - assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_SHARDS)).isEqualTo("5"); - assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); + assertThat(newIndex.getSettings().get(IndexMetadata.SETTING_NUMBER_OF_SHARDS)).isEqualTo("5"); + assertThat(newIndex.getSettings().get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); } @Test @@ -239,8 +239,8 @@ public class NewIndexTest { settings.setProperty(CLUSTER_ENABLED.getKey(), "true"); NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); - assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_SHARDS)).isEqualTo("5"); - assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("1"); + assertThat(newIndex.getSettings().get(IndexMetadata.SETTING_NUMBER_OF_SHARDS)).isEqualTo("5"); + assertThat(newIndex.getSettings().get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("1"); } @Test @@ -249,9 +249,9 @@ public class NewIndexTest { settings.setProperty("sonar.search." + index.getName() + ".shards", "3"); NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); - assertThat(newIndex.getSetting(IndexMetaData.SETTING_NUMBER_OF_SHARDS)).isEqualTo("3"); + assertThat(newIndex.getSetting(IndexMetadata.SETTING_NUMBER_OF_SHARDS)).isEqualTo("3"); // keep default value - assertThat(newIndex.getSetting(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); + assertThat(newIndex.getSetting(IndexMetadata.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); } @Test @@ -259,7 +259,7 @@ public class NewIndexTest { public void default_number_of_replicas_on_standalone_instance_must_be_0(Index index) { NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); - assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); + assertThat(newIndex.getSettings().get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); } @Test @@ -268,7 +268,7 @@ public class NewIndexTest { settings.setProperty(CLUSTER_ENABLED.getKey(), "false"); NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); - assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); + assertThat(newIndex.getSettings().get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); } @Test @@ -277,7 +277,7 @@ public class NewIndexTest { settings.setProperty(CLUSTER_ENABLED.getKey(), "true"); NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); - assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("1"); + assertThat(newIndex.getSettings().get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("1"); } @Test @@ -287,7 +287,7 @@ public class NewIndexTest { settings.setProperty(SEARCH_REPLICAS.getKey(), "0"); NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); - assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); + assertThat(newIndex.getSettings().get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); } @Test @@ -297,7 +297,7 @@ public class NewIndexTest { settings.setProperty(SEARCH_REPLICAS.getKey(), "3"); NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); - assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("3"); + assertThat(newIndex.getSettings().get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("3"); } @Test diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyClearCacheRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyClearCacheRequestBuilderTest.java deleted file mode 100644 index e3974c80d44..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyClearCacheRequestBuilderTest.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequestBuilder; -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class ProxyClearCacheRequestBuilderTest { - - @Rule - public EsTester es = EsTester.create(); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void clear_cache() { - ClearIndicesCacheRequestBuilder requestBuilder = es.client().prepareClearCache(); - requestBuilder.get(); - } - - @Test - public void to_string() { - assertThat(es.client().prepareClearCache().toString()).isEqualTo("ES clear cache request"); - assertThat(es.client().prepareClearCache("rules").toString()).isEqualTo("ES clear cache request on indices 'rules'"); - assertThat(es.client().prepareClearCache().setFields("key").toString()).isEqualTo("ES clear cache request on fields 'key'"); - assertThat(es.client().prepareClearCache().setFieldDataCache(true).toString()).isEqualTo("ES clear cache request with field data cache"); - assertThat(es.client().prepareClearCache().setRequestCache(true).toString()).isEqualTo("ES clear cache request with request cache"); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - ClearIndicesCacheRequestBuilder requestBuilder = es.client().prepareClearCache(); - requestBuilder.get(); - - assertThat(logTester.logs()).hasSize(1); - } - - @Test - public void no_trace_logs() { - logTester.setLevel(LoggerLevel.DEBUG); - ClearIndicesCacheRequestBuilder requestBuilder = es.client().prepareClearCache(); - requestBuilder.get(); - - assertThat(logTester.logs()).isEmpty(); - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().prepareClearCache().get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareClearCache().get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().prepareClearCache().execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } - -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilderTest.java deleted file mode 100644 index 4d051019190..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilderTest.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import java.util.Locale; -import java.util.Random; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.Index; - -import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class ProxyCreateIndexRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void create_index() { - CreateIndexRequestBuilder requestBuilder = es.client().prepareCreate(generateNewIndex()); - requestBuilder.get(); - } - - @Test - public void to_string() { - Index index = generateNewIndex(); - assertThat(es.client().prepareCreate(index).toString()).contains("ES create index '" + index.getName() + "'"); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - - CreateIndexRequestBuilder requestBuilder = es.client().prepareCreate(generateNewIndex()); - requestBuilder.get(); - assertThat(logTester.logs()).hasSize(1); - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().prepareCreate(generateNewIndex()).get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareCreate(generateNewIndex()).get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().prepareCreate(generateNewIndex()).execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } - - private static Index generateNewIndex(){ - String name = randomAlphabetic(10).toLowerCase(Locale.ENGLISH); - return new Random().nextBoolean() ? Index.simple(name) : Index.withRelations(name); - } - -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyDeleteRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyDeleteRequestBuilderTest.java deleted file mode 100644 index f0b5da674a9..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyDeleteRequestBuilderTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.Index; -import org.sonar.server.es.IndexType; -import org.sonar.server.es.newindex.FakeIndexDefinition; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class ProxyDeleteRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(new FakeIndexDefinition()); - @Rule - public LogTester logTester = new LogTester(); - - private final Index index = Index.simple("fakes"); - private final IndexType.IndexMainType mainType = IndexType.main(index, "fake"); - - @Test - public void delete() { - es.client().prepareDelete(mainType, "the_id").get(); - } - - @Test - public void to_string() { - assertThat(es.client().prepareDelete(mainType, "the_id").toString()).isEqualTo("ES delete request of doc the_id in index fakes/fake"); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - es.client().prepareDelete(mainType, "the_id").get(); - - assertThat(logTester.logs()).hasSize(1); - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().prepareDelete(mainType, "the_id").get("1"); - fail(); - } catch (UnsupportedOperationException e) { - assertThat(e).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareDelete(mainType, "the_id").get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (UnsupportedOperationException e) { - assertThat(e).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().prepareDelete(mainType, "the_id").execute(); - fail(); - } catch (UnsupportedOperationException e) { - assertThat(e).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyGetRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyGetRequestBuilderTest.java deleted file mode 100644 index 9c92ee0d1af..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyGetRequestBuilderTest.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import com.tngtech.java.junit.dataprovider.DataProvider; -import com.tngtech.java.junit.dataprovider.DataProviderRunner; -import com.tngtech.java.junit.dataprovider.UseDataProvider; -import org.elasticsearch.action.get.GetRequestBuilder; -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.Index; -import org.sonar.server.es.newindex.FakeIndexDefinition; -import org.sonar.server.es.IndexType; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; -import static org.sonar.server.es.newindex.FakeIndexDefinition.TYPE_FAKE; - -@RunWith(DataProviderRunner.class) -public class ProxyGetRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(new FakeIndexDefinition()); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - - es.client().prepareGet(TYPE_FAKE, "ruleKey") - .get(); - assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1); - } - - @Test - @UseDataProvider("mainAndRelationWithUnknownIndex") - public void prepareGet_fails_if_index_unknown(IndexType indexType) { - GetRequestBuilder requestBuilder = es.client().prepareGet(indexType, "rule1"); - try { - requestBuilder.get(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class); - assertThat(e.getMessage()).contains("Fail to execute ES get request for key 'rule1' on index 'unknown' on type 'test'"); - } - } - - @DataProvider - public static Object[][] mainAndRelationWithUnknownIndex() { - IndexType.IndexMainType mainType = IndexType.main(Index.withRelations("unknown"), "test"); - return new Object[][] { - {mainType}, - {IndexType.relation(mainType, "donut")} - }; - } - - @Test - public void get_with_string_timeout_is_not_implemented() { - try { - es.client().prepareGet(TYPE_FAKE, "ruleKey").get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareGet(TYPE_FAKE, "ruleKey").get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().prepareGet(TYPE_FAKE, "ruleKey").execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } - -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndexRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndexRequestBuilderTest.java deleted file mode 100644 index 1e91d8c551f..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndexRequestBuilderTest.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import com.tngtech.java.junit.dataprovider.DataProvider; -import com.tngtech.java.junit.dataprovider.DataProviderRunner; -import com.tngtech.java.junit.dataprovider.UseDataProvider; -import org.elasticsearch.action.DocWriteResponse.Result; -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.index.IndexResponse; -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.Index; -import org.sonar.server.es.newindex.FakeIndexDefinition; -import org.sonar.server.es.IndexType; -import org.sonar.server.es.IndexType.IndexMainType; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; -import static org.sonar.server.es.newindex.FakeIndexDefinition.TYPE_FAKE; - -@RunWith(DataProviderRunner.class) -public class ProxyIndexRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(new FakeIndexDefinition()); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void index_with_index_type_and_id() { - IndexResponse response = es.client().prepareIndex(TYPE_FAKE) - .setSource(FakeIndexDefinition.newDoc(42).getFields()) - .get(); - assertThat(response.getResult()).isSameAs(Result.CREATED); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - IndexResponse response = es.client().prepareIndex(TYPE_FAKE) - .setSource(FakeIndexDefinition.newDoc(42).getFields()) - .get(); - assertThat(response.getResult()).isSameAs(Result.CREATED); - assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1); - } - - @Test - @UseDataProvider("mainOrRelationType") - public void fail_if_bad_query(IndexType indexType) { - IndexRequestBuilder requestBuilder = es.client().prepareIndex(indexType); - try { - requestBuilder.get(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class); - assertThat(e.getMessage()).contains("Fail to execute ES index request for key 'null' on index 'foo' on type 'bar'"); - } - } - - @DataProvider - public static Object[][] mainOrRelationType() { - IndexMainType mainType = IndexType.main(Index.withRelations("foo"), "bar"); - return new Object[][] { - {mainType}, - {IndexType.relation(mainType, "donut")} - }; - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().prepareIndex(TYPE_FAKE).get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareIndex(TYPE_FAKE).get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void do_not_support_execute_method() { - try { - es.client().prepareIndex(TYPE_FAKE).execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } - -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndicesExistsRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndicesExistsRequestBuilderTest.java deleted file mode 100644 index 41d9ac3fe78..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndicesExistsRequestBuilderTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.Index; -import org.sonar.server.es.newindex.FakeIndexDefinition; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class ProxyIndicesExistsRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(new FakeIndexDefinition()); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void exists() { - assertThat(es.client().prepareIndicesExist(FakeIndexDefinition.DESCRIPTOR).get().isExists()).isTrue(); - assertThat(es.client().prepareIndicesExist(Index.simple("unknown")).get().isExists()).isFalse(); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - - es.client().prepareIndicesExist(FakeIndexDefinition.DESCRIPTOR).get(); - - assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1); - } - - @Test - public void to_string() { - assertThat(es.client().prepareIndicesExist(FakeIndexDefinition.DESCRIPTOR).toString()).isEqualTo("ES indices exists request on indices 'fakes'"); - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().prepareIndicesExist(FakeIndexDefinition.DESCRIPTOR).get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareIndicesExist(FakeIndexDefinition.DESCRIPTOR).get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().prepareIndicesExist(FakeIndexDefinition.DESCRIPTOR).execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } - -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilderTest.java deleted file mode 100644 index a366b7889a5..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilderTest.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.Index; -import org.sonar.server.es.newindex.FakeIndexDefinition; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class ProxyIndicesStatsRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(new FakeIndexDefinition()); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void stats() { - es.client().prepareStats(FakeIndexDefinition.DESCRIPTOR).get(); - } - - @Test - public void to_string() { - assertThat(es.client().prepareStats(FakeIndexDefinition.DESCRIPTOR).setIndices("rules").toString()).isEqualTo("ES indices stats request on indices 'rules'"); - assertThat(es.client().prepareStats().toString()).isEqualTo("ES indices stats request"); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - - es.client().prepareStats(FakeIndexDefinition.DESCRIPTOR).get(); - - assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1); - } - - @Test - public void fail_to_stats() { - try { - es.client().prepareStats(Index.simple("unknown")).get(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class); - assertThat(e.getMessage()).contains("Fail to execute ES indices stats request on indices 'unknown'"); - } - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().prepareStats(FakeIndexDefinition.DESCRIPTOR).get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareStats(FakeIndexDefinition.DESCRIPTOR).get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().prepareStats(FakeIndexDefinition.DESCRIPTOR).execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } - -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyNodesStatsRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyNodesStatsRequestBuilderTest.java deleted file mode 100644 index 1c964603d52..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyNodesStatsRequestBuilderTest.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.newindex.FakeIndexDefinition; - -import static org.assertj.core.api.Assertions.assertThat; - -public class ProxyNodesStatsRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(); - - @Rule - public LogTester logTester = new LogTester(); - - @Rule - public ExpectedException thrown = ExpectedException.none(); - - @Test - public void stats() { - es.client().prepareNodesStats().get(); - } - - @Test - public void to_string() { - assertThat(es.client().prepareNodesStats().setNodesIds("node1").toString()).isEqualTo("ES nodes stats request on nodes 'node1'"); - assertThat(es.client().prepareNodesStats().toString()).isEqualTo("ES nodes stats request"); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - - es.client().prepareNodesStats().get(); - - assertThat(logTester.logs()).hasSize(1); - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - thrown.expect(IllegalStateException.class); - thrown.expectMessage("Not yet implemented"); - - es.client().prepareNodesStats(FakeIndexDefinition.INDEX).get("1"); - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - thrown.expect(IllegalStateException.class); - thrown.expectMessage("Not yet implemented"); - - es.client().prepareNodesStats(FakeIndexDefinition.INDEX).get(TimeValue.timeValueMinutes(1)); - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - thrown.expect(UnsupportedOperationException.class); - thrown.expectMessage("execute() should not be called as it's used for asynchronous"); - - es.client().prepareNodesStats(FakeIndexDefinition.INDEX).execute(); - } - -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilderTest.java deleted file mode 100644 index fe5f47700c2..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilderTest.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import com.google.common.collect.ImmutableMap; -import java.util.HashMap; -import java.util.Map; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.newindex.FakeIndexDefinition; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class ProxyPutMappingRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(new FakeIndexDefinition()); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void put_mapping() { - PutMappingRequestBuilder requestBuilder = es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR) - .setType(FakeIndexDefinition.TYPE) - .setSource(mapDomain()); - requestBuilder.get(); - } - - @Test - public void to_string() { - assertThat(es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR).setSource(mapDomain()).toString()) - .isEqualTo("ES put mapping request on indices 'fakes' with source '{\"dynamic\":false,\"_all\":{\"enabled\":false}}'"); - assertThat(es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR).setType(FakeIndexDefinition.TYPE).setSource(mapDomain()).toString()) - .isEqualTo("ES put mapping request on indices 'fakes' on type 'fake' with source '{\"dynamic\":false,\"_all\":{\"enabled\":false}}'"); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - - PutMappingRequestBuilder requestBuilder = es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR) - .setType(FakeIndexDefinition.TYPE) - .setSource(mapDomain()); - requestBuilder.get(); - - assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1); - } - - @Test - public void fail_on_bad_query() { - try { - es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR).get(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class); - assertThat(e.getMessage()).contains("Fail to execute ES put mapping request"); - } - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR).get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR).get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR).execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } - - protected static Map mapDomain() { - Map<String, Object> mapping = new HashMap<>(); - mapping.put("dynamic", false); - mapping.put("_all", ImmutableMap.of("enabled", false)); - return mapping; - } - -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyRefreshRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyRefreshRequestBuilderTest.java deleted file mode 100644 index 47da09b91cc..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyRefreshRequestBuilderTest.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder; -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.Index; -import org.sonar.server.es.newindex.FakeIndexDefinition; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class ProxyRefreshRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(new FakeIndexDefinition()); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void refresh() { - RefreshRequestBuilder requestBuilder = es.client().prepareRefresh(FakeIndexDefinition.DESCRIPTOR); - requestBuilder.get(); - } - - @Test - public void to_string() { - assertThat(es.client().prepareRefresh(FakeIndexDefinition.DESCRIPTOR).toString()).isEqualTo("ES refresh request on indices 'fakes'"); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - - RefreshRequestBuilder requestBuilder = es.client().prepareRefresh(FakeIndexDefinition.DESCRIPTOR); - requestBuilder.get(); - assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1); - } - - @Test - public void fail_to_refresh() { - try { - es.client().prepareRefresh(Index.simple("unknown")).get(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class); - assertThat(e.getMessage()).contains("Fail to execute ES refresh request on indices 'unknown'"); - } - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().prepareRefresh(FakeIndexDefinition.DESCRIPTOR).get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareRefresh(FakeIndexDefinition.DESCRIPTOR).get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().prepareRefresh(FakeIndexDefinition.DESCRIPTOR).execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } - -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxySearchRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxySearchRequestBuilderTest.java deleted file mode 100644 index 8a73f114897..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxySearchRequestBuilderTest.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.Index; -import org.sonar.server.es.newindex.FakeIndexDefinition; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class ProxySearchRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(new FakeIndexDefinition()); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void search() { - es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).get(); - } - - @Test - public void to_string() { - assertThat(es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).setTypes(FakeIndexDefinition.TYPE).toString()).contains("ES search request '") - .contains("' on indices '[fakes]' on types '[fake]'"); - assertThat(es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).toString()) - .contains("ES search request '") - .contains("' on indices '[fakes]'"); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - - es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).get(); - assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1); - } - - @Test - public void fail_to_search_bad_query() { - try { - es.client().prepareSearch(Index.simple("unknown")).get(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class); - assertThat(e.getMessage()) - .contains("Fail to execute ES search request 'SearchRequest{") - .contains("}' on indices '[unknown]'"); - } - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilderTest.java deleted file mode 100644 index fe7308f942e..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilderTest.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.newindex.FakeIndexDefinition; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class ProxySearchScrollRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(new FakeIndexDefinition()); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - - SearchResponse response = es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR) - .setScroll(TimeValue.timeValueMinutes(1)) - .get(); - logTester.clear(); - es.client().prepareSearchScroll(response.getScrollId()).get(); - assertThat(logTester.logs()).hasSize(1); - } - - @Test - public void no_trace_logs() { - logTester.setLevel(LoggerLevel.DEBUG); - - SearchResponse response = es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR) - .setScroll(TimeValue.timeValueMinutes(1)) - .get(); - logTester.clear(); - es.client().prepareSearchScroll(response.getScrollId()).get(); - assertThat(logTester.logs()).isEmpty(); - } - - @Test - public void fail_to_search_bad_query() { - try { - es.client().prepareSearchScroll("unknown").get(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class); - assertThat(e.getMessage()).contains("Fail to execute ES search scroll request for scroll id 'null'"); - } - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().prepareSearchScroll("scrollId").get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareSearchScroll("scrollId").get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().prepareSearchScroll("scrollId").execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyWebServerHealthRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyWebServerHealthRequestBuilderTest.java deleted file mode 100644 index d3b164fcbd7..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyWebServerHealthRequestBuilderTest.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class ProxyWebServerHealthRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void state() { - ClusterHealthRequestBuilder requestBuilder = es.client().prepareHealth(); - ClusterHealthResponse state = requestBuilder.get(); - assertThat(state.getStatus()).isEqualTo(ClusterHealthStatus.GREEN); - } - - @Test - public void to_string() { - assertThat(es.client().prepareHealth().toString()).isEqualTo("ES cluster health request"); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - - ClusterHealthRequestBuilder requestBuilder = es.client().prepareHealth(); - ClusterHealthResponse state = requestBuilder.get(); - assertThat(state.getStatus()).isEqualTo(ClusterHealthStatus.GREEN); - - assertThat(logTester.logs()).hasSize(1); - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().prepareHealth().get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareHealth().get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().prepareHealth().execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } - -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyWebServerStateRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyWebServerStateRequestBuilderTest.java deleted file mode 100644 index a48918ba7c7..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyWebServerStateRequestBuilderTest.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class ProxyWebServerStateRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void state() { - ClusterStateRequestBuilder requestBuilder = es.client().prepareState(); - requestBuilder.get(); - } - - @Test - public void to_string() { - assertThat(es.client().prepareState().setIndices("rules").toString()).isEqualTo("ES cluster state request on indices 'rules'"); - assertThat(es.client().prepareState().toString()).isEqualTo("ES cluster state request"); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - - ClusterStateRequestBuilder requestBuilder = es.client().prepareState(); - requestBuilder.get(); - - assertThat(logTester.logs()).hasSize(1); - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().prepareState().get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareState().get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().prepareState().execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } - -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyWebServerStatsRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyWebServerStatsRequestBuilderTest.java deleted file mode 100644 index 83173c978bd..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyWebServerStatsRequestBuilderTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2020 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequestBuilder; -import org.elasticsearch.common.unit.TimeValue; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class ProxyWebServerStatsRequestBuilderTest { - - @Rule - public EsTester es = EsTester.createCustom(); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void stats() { - ClusterStatsRequestBuilder requestBuilder = es.client().prepareClusterStats(); - requestBuilder.get(); - } - - @Test - public void to_string() { - assertThat(es.client().prepareClusterStats().setNodesIds("node1").toString()).isEqualTo("ES cluster stats request on nodes 'node1'"); - assertThat(es.client().prepareClusterStats().toString()).isEqualTo("ES cluster stats request"); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - - ClusterStatsRequestBuilder requestBuilder = es.client().prepareClusterStats(); - requestBuilder.get(); - assertThat(logTester.logs()).hasSize(1); - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - es.client().prepareClusterStats().get("1"); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - es.client().prepareClusterStats().get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - es.client().prepareClusterStats().execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } - -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/response/ClusterStatsResponseTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/response/ClusterStatsResponseTest.java new file mode 100644 index 00000000000..2682789a603 --- /dev/null +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/response/ClusterStatsResponseTest.java @@ -0,0 +1,52 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.response; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ClusterStatsResponseTest { + private static final String EXAMPLE_JSON = "{" + + " \"status\": \"yellow\"," + + " \"nodes\": {" + + " \"count\": {" + + " \"total\": 3" + + " }" + + " }" + + "}"; + + @Test + public void should_parse_example_json() { + JsonObject jsonObject = getExampleAsJsonObject(); + ClusterStatsResponse clusterStatsResponse = ClusterStatsResponse.toClusterStatsResponse(jsonObject); + + assertThat(clusterStatsResponse.getHealthStatus()).isEqualTo(ClusterHealthStatus.YELLOW); + assertThat(clusterStatsResponse.getNodeCount()).isEqualTo(3); + } + + private static JsonObject getExampleAsJsonObject() { + return new Gson().fromJson(EXAMPLE_JSON, JsonObject.class); + } + +} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/response/IndicesStatsResponseTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/response/IndicesStatsResponseTest.java new file mode 100644 index 00000000000..0ed3ac313c5 --- /dev/null +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/response/IndicesStatsResponseTest.java @@ -0,0 +1,89 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.response; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import java.util.Collection; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class IndicesStatsResponseTest { + private static final String EXAMPLE_JSON = "{" + + " \"indices\": {" + + " \"index-1\": {" + + " \"primaries\": {" + + " \"docs\": {" + + " \"count\": 1234" + + " }," + + " \"store\": {" + + " \"size_in_bytes\": 56789" + + " }" + + " }," + + " \"shards\": {" + + " \"shard-1\": {}," + + " \"shard-2\": {}" + + " }" + + " }," + + " \"index-2\": {" + + " \"primaries\": {" + + " \"docs\": {" + + " \"count\": 42" + + " }," + + " \"store\": {" + + " \"size_in_bytes\": 123" + + " }" + + " }," + + " \"shards\": {" + + " \"shard-1\": {}," + + " \"shard-2\": {}" + + " }" + + " }" + + " }" + + "}"; + + @Test + public void should_parse_example_json() { + JsonObject jsonObject = getExampleAsJsonObject(); + IndicesStatsResponse indicesStatsResponse = IndicesStatsResponse.toIndicesStatsResponse(jsonObject); + + Collection<IndexStats> allIndexStats = indicesStatsResponse.getAllIndexStats(); + assertThat(allIndexStats) + .hasSize(2) + .extracting("name") + .contains("index-1", "index-2"); + + IndexStats indexStats = allIndexStats.stream().filter(i -> i.getName().equals("index-1")).findFirst().get(); + assertThat(indexStats.getDocCount()).isEqualTo(1234); + assertThat(indexStats.getShardsCount()).isEqualTo(2); + assertThat(indexStats.getStoreSizeBytes()).isEqualTo(56789); + + indexStats = allIndexStats.stream().filter(i -> i.getName().equals("index-2")).findFirst().get(); + assertThat(indexStats.getDocCount()).isEqualTo(42); + assertThat(indexStats.getStoreSizeBytes()).isEqualTo(123); + assertThat(indexStats.getShardsCount()).isEqualTo(2); + } + + private static JsonObject getExampleAsJsonObject() { + return new Gson().fromJson(EXAMPLE_JSON, JsonObject.class); + } + +} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/response/NodeStatsResponseTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/response/NodeStatsResponseTest.java new file mode 100644 index 00000000000..6c7c6e7eef3 --- /dev/null +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/response/NodeStatsResponseTest.java @@ -0,0 +1,135 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.response; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class NodeStatsResponseTest { + private final static String EXAMPLE_JSON = "{" + + " \"nodes\": {" + + " \"YnKPZcbGRamRQGxjErLWoQ\": {" + + " \"name\": \"sonarqube\"," + + " \"host\": \"127.0.0.1\"," + + " \"indices\": {" + + " \"docs\": {" + + " \"count\": 13557" + + " }," + + " \"store\": {" + + " \"size_in_bytes\": 8670970" + + " }," + + " \"query_cache\": {" + + " \"memory_size_in_bytes\": 0" + + " }," + + " \"fielddata\": {" + + " \"memory_size_in_bytes\": 4880" + + " }," + + " \"translog\": {" + + " \"size_in_bytes\": 8274137" + + " }," + + " \"request_cache\": {" + + " \"memory_size_in_bytes\": 0" + + " }" + + " }," + + " \"process\": {" + + " \"open_file_descriptors\": 296," + + " \"max_file_descriptors\": 10240," + + " \"cpu\": {" + + " \"percent\": 7" + + " }" + + " }," + + " \"jvm\": {" + + " \"mem\": {" + + " \"heap_used_in_bytes\": 158487160," + + " \"heap_used_percent\": 30," + + " \"heap_max_in_bytes\": 518979584," + + " \"non_heap_used_in_bytes\": 109066592" + + " }," + + " \"threads\": {" + + " \"count\": 70" + + " }" + + " }," + + " \"fs\": {" + + " \"total\": {" + + " \"total_in_bytes\": 250685575168," + + " \"free_in_bytes\": 142843138048," + + " \"available_in_bytes\": 136144027648" + + " }" + + " }," + + " \"breakers\": {" + + " \"request\": {" + + " \"limit_size_in_bytes\": 311387750," + + " \"estimated_size_in_bytes\": 1" + + " }," + + " \"fielddata\": {" + + " \"limit_size_in_bytes\": 207591833," + + " \"estimated_size_in_bytes\": 4880" + + " }" + + " }" + + " }" + + " }" + + "}"; + + @Test + public void should_parse_example_json() { + JsonObject jsonObject = getExampleAsJsonObject(); + NodeStatsResponse nodeStatsResponse = NodeStatsResponse.toNodeStatsResponse(jsonObject); + + assertThat(nodeStatsResponse.getNodeStats()).hasSize(1); + + NodeStats nodeStats = nodeStatsResponse.getNodeStats().get(0); + assertThat(nodeStats.getName()).isEqualTo("sonarqube"); + assertThat(nodeStats.getHost()).isEqualTo("127.0.0.1"); + assertThat(nodeStats.getCpuUsage()).isEqualTo(7); + + assertThat(nodeStats.getOpenFileDescriptors()).isEqualTo(296); + assertThat(nodeStats.getMaxFileDescriptors()).isEqualTo(10240); + assertThat(nodeStats.getDiskAvailableBytes()).isEqualTo(136144027648L); + + assertThat(nodeStats.getFieldDataCircuitBreakerLimit()).isEqualTo(207591833); + assertThat(nodeStats.getFieldDataCircuitBreakerEstimation()).isEqualTo(4880); + assertThat(nodeStats.getRequestCircuitBreakerLimit()).isEqualTo(311387750L); + assertThat(nodeStats.getRequestCircuitBreakerEstimation()).isEqualTo(1); + + JvmStats jvmStats = nodeStats.getJvmStats(); + assertThat(jvmStats).isNotNull(); + assertThat(jvmStats.getHeapUsedPercent()).isEqualTo(30); + assertThat(jvmStats.getHeapUsedInBytes()).isEqualTo(158487160); + assertThat(jvmStats.getHeapMaxInBytes()).isEqualTo(518979584); + assertThat(jvmStats.getNonHeapUsedInBytes()).isEqualTo(109066592); + assertThat(jvmStats.getThreadCount()).isEqualTo(70); + + IndicesStats indicesStats = nodeStats.getIndicesStats(); + assertThat(indicesStats).isNotNull(); + assertThat(indicesStats.getStoreSizeInBytes()).isEqualTo(8670970); + assertThat(indicesStats.getTranslogSizeInBytes()).isEqualTo(8274137); + assertThat(indicesStats.getRequestCacheMemorySizeInBytes()).isEqualTo(0); + assertThat(indicesStats.getFieldDataMemorySizeInBytes()).isEqualTo(4880); + assertThat(indicesStats.getQueryCacheMemorySizeInBytes()).isEqualTo(0); + + } + + private static JsonObject getExampleAsJsonObject() { + return new Gson().fromJson(EXAMPLE_JSON, JsonObject.class); + } +} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/TopAggregationHelperTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/TopAggregationHelperTest.java index 1d615d974f0..dbb76a0d77d 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/TopAggregationHelperTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/TopAggregationHelperTest.java @@ -29,7 +29,7 @@ import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder; import org.junit.Test; import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexerTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexerTest.java index da52873dbf5..d054d06760d 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexerTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexerTest.java @@ -23,8 +23,9 @@ import java.util.Arrays; import java.util.Collection; import java.util.function.Consumer; import java.util.function.Predicate; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.Rule; import org.junit.Test; import org.sonar.api.resources.Qualifiers; @@ -50,6 +51,7 @@ import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.index.query.QueryBuilders.termsQuery; import static org.sonar.db.component.ComponentTesting.newPrivateProjectDto; +import static org.sonar.server.es.EsClient.prepareSearch; import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE; import static org.sonar.server.es.ProjectIndexer.Cause.PROJECT_CREATION; import static org.sonar.server.es.ProjectIndexer.Cause.PROJECT_DELETION; @@ -288,12 +290,13 @@ public class ProjectMeasuresIndexerTest { } private void assertThatProjectHasTag(ComponentDto project, String expectedTag) { - SearchRequestBuilder request = es.client() - .prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) - .setQuery(boolQuery() - .filter(termQuery(FIELD_INDEX_TYPE, TYPE_PROJECT_MEASURES.getName())) - .filter(termQuery(FIELD_TAGS, expectedTag))); - assertThat(request.get().getHits().getHits()) + SearchRequest request = prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) + .source(new SearchSourceBuilder() + .query(boolQuery() + .filter(termQuery(FIELD_INDEX_TYPE, TYPE_PROJECT_MEASURES.getName())) + .filter(termQuery(FIELD_TAGS, expectedTag)))); + + assertThat(es.client().search(request).getHits().getHits()) .extracting(SearchHit::getId) .contains(project.uuid()); } @@ -323,13 +326,15 @@ public class ProjectMeasuresIndexerTest { } private void assertThatQualifierIs(String qualifier, String... componentsUuid) { - SearchRequestBuilder request = es.client() - .prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) - .setQuery(boolQuery() + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .query(boolQuery() .filter(termQuery(FIELD_INDEX_TYPE, TYPE_PROJECT_MEASURES.getName())) .filter(termQuery(FIELD_QUALIFIER, qualifier)) .filter(termsQuery(FIELD_UUID, componentsUuid))); - assertThat(request.get().getHits().getHits()) + + SearchRequest request = prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) + .source(searchSourceBuilder); + assertThat(es.client().search(request).getHits().getHits()) .extracting(SearchHit::getId) .containsExactlyInAnyOrder(componentsUuid); } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexDefinitionTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexDefinitionTest.java index 33cc879982a..25b94cbec77 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexDefinitionTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexDefinitionTest.java @@ -20,12 +20,19 @@ package org.sonar.server.rule.index; import com.google.common.collect.ImmutableMap; +import java.io.IOException; import java.util.List; import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; +import org.apache.lucene.search.TotalHits; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.indices.AnalyzeRequest; +import org.elasticsearch.client.indices.AnalyzeResponse; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.Rule; import org.junit.Test; import org.sonar.api.config.internal.MapSettings; +import org.sonar.server.es.EsClient; import org.sonar.server.es.EsTester; import org.sonar.server.es.Index; import org.sonar.server.es.IndexDefinition; @@ -93,8 +100,10 @@ public class RuleIndexDefinitionTest { FIELD_RULE_REPOSITORY, "squid", FIELD_RULE_KEY, "squid:S001"))); assertThat(tester.countDocuments(TYPE_RULE)).isEqualTo(1); - assertThat(tester.client().prepareSearch(TYPE_RULE.getIndex()).setQuery(matchQuery(ENGLISH_HTML_ANALYZER.subField(FIELD_RULE_HTML_DESCRIPTION), "brown fox jumps lazy")) - .get().getHits().getTotalHits()).isEqualTo(1); + assertThat(tester.client().search(EsClient.prepareSearch(TYPE_RULE) + .source(new SearchSourceBuilder() + .query(matchQuery(ENGLISH_HTML_ANALYZER.subField(FIELD_RULE_HTML_DESCRIPTION), "brown fox jumps lazy")))) + .getHits().getTotalHits()).isEqualTo(new TotalHits(1, TotalHits.Relation.EQUAL_TO)); } @Test @@ -115,9 +124,12 @@ public class RuleIndexDefinitionTest { } private List<AnalyzeResponse.AnalyzeToken> analyzeIndexedTokens(String text) { - return tester.client().nativeClient().admin().indices().prepareAnalyze(TYPE_RULE.getIndex().getName(), - text) - .setField(ENGLISH_HTML_ANALYZER.subField(FIELD_RULE_HTML_DESCRIPTION)) - .execute().actionGet().getTokens(); + try { + return tester.nativeClient().indices() + .analyze(AnalyzeRequest.withField(TYPE_RULE.getIndex().getName(), ENGLISH_HTML_ANALYZER.subField(FIELD_RULE_HTML_DESCRIPTION), text), RequestOptions.DEFAULT) + .getTokens(); + } catch (IOException e) { + throw new IllegalStateException("Could not analyze indexed tokens for text: " + text); + } } } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java index dc0b9bbd56a..b44c1e120e5 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java @@ -29,6 +29,8 @@ import java.util.Set; import java.util.stream.IntStream; import java.util.stream.Stream; import javax.annotation.Nullable; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -45,6 +47,7 @@ import org.sonar.db.rule.RuleDefinitionDto; import org.sonar.db.rule.RuleDto; import org.sonar.db.rule.RuleDto.Scope; import org.sonar.db.rule.RuleTesting; +import org.sonar.server.es.EsClient; import org.sonar.server.es.EsTester; import org.sonar.server.security.SecurityStandards; import org.sonar.server.security.SecurityStandards.SQCategory; diff --git a/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/EsTester.java b/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/EsTester.java index 86408377761..da9a62d9da3 100644 --- a/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/EsTester.java +++ b/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/EsTester.java @@ -30,6 +30,7 @@ import java.net.ServerSocket; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -40,18 +41,29 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.lang.reflect.ConstructorUtils; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.apache.http.HttpHost; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.ClearScrollRequest; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.analysis.common.CommonAnalysisPlugin; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.client.indices.CreateIndexResponse; +import org.elasticsearch.client.indices.GetIndexRequest; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.discovery.DiscoveryModule; @@ -59,8 +71,8 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.http.BindHttpException; import org.elasticsearch.http.HttpTransportSettings; -import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.join.ParentJoinPlugin; @@ -68,6 +80,8 @@ import org.elasticsearch.node.InternalSettingsPreparer; import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeValidationException; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.transport.Netty4Plugin; import org.junit.rules.ExternalResource; import org.sonar.api.utils.log.Logger; @@ -87,6 +101,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.Lists.newArrayList; import static java.lang.String.format; +import static org.assertj.core.api.Assertions.assertThat; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.sonar.server.es.Index.ALL_INDICES; import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE; @@ -114,6 +129,8 @@ public class EsTester extends ExternalResource { } private static final Node SHARED_NODE = createNode(); + private static final EsClient ES_REST_CLIENT = createEsRestClient(SHARED_NODE); + private static final AtomicBoolean CORE_INDICES_CREATED = new AtomicBoolean(false); private static final Set<String> CORE_INDICES_NAMES = new HashSet<>(); @@ -155,26 +172,57 @@ public class EsTester extends ExternalResource { protected void after() { if (isCustom) { // delete non-core indices - String[] existingIndices = SHARED_NODE.client().admin().indices().prepareGetIndex().get().getIndices(); + String[] existingIndices = getIndicesNames(); Stream.of(existingIndices) .filter(i -> !CORE_INDICES_NAMES.contains(i)) .forEach(EsTester::deleteIndexIfExists); } - BulkIndexer.delete(client(), IndexType.main(ALL_INDICES, "dummy"), client().prepareSearch(ALL_INDICES).setQuery(matchAllQuery())); + + BulkIndexer.delete(ES_REST_CLIENT, IndexType.main(ALL_INDICES, "dummy"), + EsClient.prepareSearch(ALL_INDICES.getName()) + .source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()))); + } + + private static String[] getIndicesNames() { + String[] existingIndices; + try { + existingIndices = ES_REST_CLIENT.nativeClient().indices().get(new GetIndexRequest(ALL_INDICES.getName()), RequestOptions.DEFAULT).getIndices(); + } catch (ElasticsearchStatusException e) { + if (e.status().getStatus() == 404) { + existingIndices = new String[0]; + } else { + throw e; + } + } catch (IOException e) { + throw new IllegalStateException("Could not get indicies", e); + } + return existingIndices; + } + + private static EsClient createEsRestClient(Node sharedNode) { + assertThat(sharedNode.isClosed()).isFalse(); + + String host = sharedNode.settings().get(HttpTransportSettings.SETTING_HTTP_BIND_HOST.getKey()); + Integer port = sharedNode.settings().getAsInt(HttpTransportSettings.SETTING_HTTP_PORT.getKey(), -1); + return new EsClient(new HttpHost(host, port)); } public EsClient client() { - return new EsClient(SHARED_NODE.client()); + return ES_REST_CLIENT; + } + + public RestHighLevelClient nativeClient() { + return ES_REST_CLIENT.nativeClient(); } public void putDocuments(IndexType indexType, BaseDoc... docs) { try { - BulkRequestBuilder bulk = SHARED_NODE.client().prepareBulk() + BulkRequest bulk = new BulkRequest() .setRefreshPolicy(REFRESH_IMMEDIATE); for (BaseDoc doc : docs) { bulk.add(doc.toIndexRequest()); } - BulkResponse bulkResponse = bulk.get(); + BulkResponse bulkResponse = ES_REST_CLIENT.bulk(bulk); if (bulkResponse.hasFailures()) { throw new IllegalStateException(bulkResponse.buildFailureMessage()); } @@ -185,14 +233,14 @@ public class EsTester extends ExternalResource { public void putDocuments(IndexType indexType, Map<String, Object>... docs) { try { - BulkRequestBuilder bulk = SHARED_NODE.client().prepareBulk() + BulkRequest bulk = new BulkRequest() .setRefreshPolicy(REFRESH_IMMEDIATE); for (Map<String, Object> doc : docs) { IndexType.IndexMainType mainType = indexType.getMainType(); bulk.add(new IndexRequest(mainType.getIndex().getName(), mainType.getType()) .source(doc)); } - BulkResponse bulkResponse = bulk.get(); + BulkResponse bulkResponse = ES_REST_CLIENT.bulk(bulk); if (bulkResponse.hasFailures()) { throw new IllegalStateException(bulkResponse.buildFailureMessage()); } @@ -202,15 +250,23 @@ public class EsTester extends ExternalResource { } public long countDocuments(Index index) { - return client().prepareSearch(index) - .setQuery(matchAllQuery()) - .setSize(0).get().getHits().getTotalHits(); + SearchRequest searchRequest = EsClient.prepareSearch(index.getName()) + .source(new SearchSourceBuilder() + .query(QueryBuilders.matchAllQuery()) + .size(0)); + + return ES_REST_CLIENT.search(searchRequest) + .getHits().getTotalHits().value; } public long countDocuments(IndexType indexType) { - return client().prepareSearch(indexType.getMainType()) - .setQuery(getDocumentsQuery(indexType)) - .setSize(0).get().getHits().getTotalHits(); + SearchRequest searchRequest = EsClient.prepareSearch(indexType.getMainType()) + .source(new SearchSourceBuilder() + .query(getDocumentsQuery(indexType)) + .size(0)); + + return ES_REST_CLIENT.search(searchRequest) + .getHits().getTotalHits().value; } /** @@ -229,38 +285,33 @@ public class EsTester extends ExternalResource { } /** - * Get all the indexed documents (no paginated results) in the specified index, whatever their type. Results are not sorted. - */ - public List<SearchHit> getDocuments(Index index) { - SearchRequestBuilder req = SHARED_NODE.client() - .prepareSearch(index.getName()) - .setQuery(matchAllQuery()); - return getDocuments(req); - } - - /** * Get all the indexed documents (no paginated results) of the specified type. Results are not sorted. */ public List<SearchHit> getDocuments(IndexType indexType) { IndexType.IndexMainType mainType = indexType.getMainType(); - SearchRequestBuilder req = SHARED_NODE.client() - .prepareSearch(mainType.getIndex().getName()) - .setQuery(getDocumentsQuery(indexType)); - return getDocuments(req); + + SearchRequest searchRequest = EsClient.prepareSearch(mainType.getIndex().getName()) + .source(new SearchSourceBuilder() + .query(getDocumentsQuery(indexType))); + return getDocuments(searchRequest); } - private List<SearchHit> getDocuments(SearchRequestBuilder req) { - EsUtils.optimizeScrollRequest(req); - req.setScroll(new TimeValue(60000)) - .setSize(100); + private List<SearchHit> getDocuments(SearchRequest req) { + req.scroll(new TimeValue(60000)); + req.source() + .size(100) + .sort("_doc", SortOrder.ASC); - SearchResponse response = req.get(); + SearchResponse response = ES_REST_CLIENT.search(req); List<SearchHit> result = newArrayList(); while (true) { Iterables.addAll(result, response.getHits()); - response = SHARED_NODE.client().prepareSearchScroll(response.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet(); + response = ES_REST_CLIENT.scroll(new SearchScrollRequest(response.getScrollId()).scroll(new TimeValue(600000))); // Break condition: No hits are returned if (response.getHits().getHits().length == 0) { + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); + clearScrollRequest.addScrollId(response.getScrollId()); + ES_REST_CLIENT.clearScroll(clearScrollRequest); break; } } @@ -304,19 +355,28 @@ public class EsTester extends ExternalResource { } private void setIndexSettings(String index, Map<String, Object> settings) { - AcknowledgedResponse response = SHARED_NODE.client().admin().indices() - .prepareUpdateSettings(index) - .setSettings(settings) - .get(); + AcknowledgedResponse response = null; + try { + response = ES_REST_CLIENT.nativeClient().indices() + .putSettings(new UpdateSettingsRequest(index).settings(settings), RequestOptions.DEFAULT); + } catch (IOException e) { + throw new IllegalStateException("Could not update index settings", e); + } checkState(response.isAcknowledged()); } private static void deleteIndexIfExists(String name) { try { - AcknowledgedResponse response = SHARED_NODE.client().admin().indices().prepareDelete(name).get(); + AcknowledgedResponse response = ES_REST_CLIENT.nativeClient().indices().delete(new DeleteIndexRequest(name), RequestOptions.DEFAULT); checkState(response.isAcknowledged(), "Fail to drop the index " + name); - } catch (IndexNotFoundException e) { - // ignore + } catch (ElasticsearchStatusException e) { + if (e.status().getStatus() == 404) { + // ignore, index not found + } else { + throw e; + } + } catch (IOException e) { + throw new IllegalStateException("Could not delete index", e); } } @@ -334,28 +394,57 @@ public class EsTester extends ExternalResource { // create index Settings.Builder settings = Settings.builder(); settings.put(index.getSettings()); - CreateIndexResponse indexResponse = SHARED_NODE.client().admin().indices() - .prepareCreate(indexName) - .setSettings(settings) - .get(); + + CreateIndexResponse indexResponse = createIndex(indexName, settings); + if (!indexResponse.isAcknowledged()) { throw new IllegalStateException("Failed to create index " + indexName); } - SHARED_NODE.client().admin().cluster().prepareHealth(indexName).setWaitForStatus(ClusterHealthStatus.YELLOW).get(); + + waitForClusterYellowStatus(indexName); // create types String typeName = index.getMainType().getType(); - AcknowledgedResponse mappingResponse = SHARED_NODE.client().admin().indices().preparePutMapping(indexName) - .setType(typeName) - .setSource(index.getAttributes()) - .get(); + putIndexMapping(index, indexName, typeName); + + waitForClusterYellowStatus(indexName); + + result.add(index); + } + return result; + } + + private static void waitForClusterYellowStatus(String indexName) { + try { + ES_REST_CLIENT.nativeClient().cluster().health(new ClusterHealthRequest(indexName).waitForStatus(ClusterHealthStatus.YELLOW), RequestOptions.DEFAULT); + } catch (IOException e) { + throw new IllegalStateException("Could not query for index health status"); + } + } + + private static void putIndexMapping(BuiltIndex index, String indexName, String typeName) { + try { + AcknowledgedResponse mappingResponse = ES_REST_CLIENT.nativeClient().indices().putMapping(new PutMappingRequest(indexName) + .type(typeName) + .source(index.getAttributes()), RequestOptions.DEFAULT); + if (!mappingResponse.isAcknowledged()) { throw new IllegalStateException("Failed to create type " + typeName); } - SHARED_NODE.client().admin().cluster().prepareHealth(indexName).setWaitForStatus(ClusterHealthStatus.YELLOW).get(); - result.add(index); + } catch (IOException e) { + throw new IllegalStateException("Could not query for put index mapping"); } - return result; + } + + private static CreateIndexResponse createIndex(String indexName, Settings.Builder settings) { + CreateIndexResponse indexResponse; + try { + indexResponse = ES_REST_CLIENT.nativeClient().indices() + .create(new CreateIndexRequest(indexName).settings(settings), RequestOptions.DEFAULT); + } catch (IOException e) { + throw new IllegalStateException("Could not create index"); + } + return indexResponse; } private static Node createNode() { @@ -393,12 +482,11 @@ public class EsTester extends ExternalResource { .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "1b") // always reduce this - it can make tests really slow .put(RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC_SETTING.getKey(), TimeValue.timeValueMillis(20)) - .put(NetworkModule.HTTP_ENABLED.getKey(), true) .put(HttpTransportSettings.SETTING_HTTP_PORT.getKey(), httpPort) .put(HttpTransportSettings.SETTING_HTTP_BIND_HOST.getKey(), "localhost") .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "single-node") .build(); - Node node = new Node(InternalSettingsPreparer.prepareEnvironment(settings, null), + Node node = new Node(InternalSettingsPreparer.prepareEnvironment(settings, Collections.emptyMap(), null, null), ImmutableList.of( CommonAnalysisPlugin.class, // Netty4Plugin provides http and tcp transport @@ -406,10 +494,6 @@ public class EsTester extends ExternalResource { // install ParentJoin plugin required to create field of type "join" ParentJoinPlugin.class), true) { - @Override - protected void registerDerivedNodeNameWithLogger(String nodeName) { - // nothing to do - } }; return node.start(); } diff --git a/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/newindex/FakeIndexDefinition.java b/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/newindex/FakeIndexDefinition.java index 748e813ae25..7c980ba5b96 100644 --- a/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/newindex/FakeIndexDefinition.java +++ b/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/newindex/FakeIndexDefinition.java @@ -19,7 +19,7 @@ */ package org.sonar.server.es.newindex; -import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.sonar.api.config.internal.MapSettings; import org.sonar.server.es.FakeDoc; import org.sonar.server.es.Index; @@ -47,7 +47,7 @@ public class FakeIndexDefinition implements IndexDefinition { @Override public void define(IndexDefinitionContext context) { NewIndex index = context.create(DESCRIPTOR, newBuilder(new MapSettings().asConfig()).build()); - index.getSettings().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, replicas); + index.getSettings().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, replicas); index.getSettings().put("index.refresh_interval", "-1"); index.createTypeMapping(TYPE_FAKE) .createIntegerField(INT_FIELD); diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/es/IndexerStartupTask.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/es/IndexerStartupTask.java index 8896b4f6869..955798333e9 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/es/IndexerStartupTask.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/es/IndexerStartupTask.java @@ -21,9 +21,7 @@ package org.sonar.server.es; import java.util.Set; import java.util.stream.Collectors; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.sonar.api.config.Configuration; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; @@ -108,8 +106,9 @@ public class IndexerStartupTask { } private void waitForIndexYellow(String index) { - Client nativeClient = esClient.nativeClient(); - ClusterHealthAction.INSTANCE.newRequestBuilder(nativeClient).setIndices(index).setWaitForYellowStatus().get(TimeValue.timeValueMinutes(10)); + esClient.clusterHealth(new ClusterHealthRequest() + .indices(index) + .waitForYellowStatus()); } private static String getSynchronousIndexingLogMessage(Set<IndexType> emptyTypes) { diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/es/MigrationEsClientImpl.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/es/MigrationEsClientImpl.java index 7d093a2f7e1..9cbafb12d11 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/es/MigrationEsClientImpl.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/es/MigrationEsClientImpl.java @@ -26,7 +26,9 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.elasticsearch.action.admin.indices.stats.IndexStats; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.client.indices.GetIndexRequest; import org.sonar.api.utils.log.Loggers; import org.sonar.core.util.stream.MoreCollectors; import org.sonar.server.platform.db.migration.es.MigrationEsClient; @@ -41,8 +43,8 @@ public class MigrationEsClientImpl implements MigrationEsClient { @Override public void deleteIndexes(String name, String... otherNames) { - Map<String, IndexStats> indices = client.nativeClient().admin().indices().prepareStats().get().getIndices(); - Set<String> existingIndices = indices.values().stream().map(IndexStats::getIndex).collect(MoreCollectors.toSet()); + String[] indices = client.getIndex(new GetIndexRequest("_all")).getIndices(); + Set<String> existingIndices = Arrays.stream(indices).collect(MoreCollectors.toSet()); Stream.concat(Stream.of(name), Arrays.stream(otherNames)) .distinct() .filter(existingIndices::contains) @@ -51,17 +53,18 @@ public class MigrationEsClientImpl implements MigrationEsClient { @Override public void addMappingToExistingIndex(String index, String type, String mappingName, String mappingType, Map<String, String> options) { - IndexStats stats = client.nativeClient().admin().indices().prepareStats().get().getIndex(index); - if (stats != null) { + String[] indices = client.getIndex(new GetIndexRequest(index)).getIndices(); + if (indices != null && indices.length == 1) { Loggers.get(getClass()).info("Add mapping [{}] to Elasticsearch index [{}]", mappingName, index); String mappingOptions = Stream.concat(Stream.of(Maps.immutableEntry("type", mappingType)), options.entrySet().stream()) .map(e -> e.getKey() + "=" + e.getValue()) .collect(Collectors.joining(",")); - client.nativeClient().admin().indices().preparePutMapping(index) - .setType(type) - .setSource(mappingName, mappingOptions) - .get(); + client.putMapping(new PutMappingRequest(index) + .type(type) + .source(mappingName, mappingOptions)); updatedIndices.add(index); + } else { + throw new IllegalStateException("Expected only one index to be found, actual [" + String.join(",", indices) + "]"); } } @@ -72,6 +75,6 @@ public class MigrationEsClientImpl implements MigrationEsClient { private void deleteIndex(String index) { Loggers.get(getClass()).info("Drop Elasticsearch index [{}]", index); - client.nativeClient().admin().indices().prepareDelete(index).get(); + client.deleteIndex(new DeleteIndexRequest(index)); } } diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsIndexesSection.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsIndexesSection.java index 7d0314480d5..f30781dae33 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsIndexesSection.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsIndexesSection.java @@ -19,16 +19,15 @@ */ package org.sonar.server.platform.monitoring; -import java.util.Map; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.admin.indices.stats.IndexStats; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.sonar.api.server.ServerSide; import org.sonar.api.utils.log.Loggers; import org.sonar.process.systeminfo.Global; import org.sonar.process.systeminfo.SystemInfoSection; import org.sonar.process.systeminfo.protobuf.ProtobufSystemInfo; import org.sonar.server.es.EsClient; +import org.sonar.server.es.response.IndexStats; +import org.sonar.server.es.response.IndicesStatsResponse; import static org.apache.commons.io.FileUtils.byteCountToDisplaySize; import static org.sonar.process.systeminfo.SystemInfoUtils.setAttribute; @@ -56,12 +55,12 @@ public class EsIndexesSection implements SystemInfoSection, Global { } private void completeIndexAttributes(ProtobufSystemInfo.Section.Builder protobuf) { - IndicesStatsResponse indicesStats = esClient.prepareStats().all().get(); - for (Map.Entry<String, IndexStats> indexStats : indicesStats.getIndices().entrySet()) { - String prefix = "Index " + indexStats.getKey() + " - "; - setAttribute(protobuf, prefix + "Docs", indexStats.getValue().getPrimaries().getDocs().getCount()); - setAttribute(protobuf, prefix + "Shards", indexStats.getValue().getShards().length); - setAttribute(protobuf, prefix + "Store Size", byteCountToDisplaySize(indexStats.getValue().getPrimaries().getStore().getSizeInBytes())); + IndicesStatsResponse indicesStats = esClient.indicesStats(); + for (IndexStats indexStats : indicesStats.getAllIndexStats()) { + String prefix = "Index " + indexStats.getName() + " - "; + setAttribute(protobuf, prefix + "Docs", indexStats.getDocCount()); + setAttribute(protobuf, prefix + "Shards", indexStats.getShardsCount()); + setAttribute(protobuf, prefix + "Store Size", byteCountToDisplaySize(indexStats.getStoreSizeBytes())); } } } diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsStateSection.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsStateSection.java index ebfa033e44d..aee69ff7b20 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsStateSection.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsStateSection.java @@ -21,15 +21,14 @@ package org.sonar.server.platform.monitoring; import java.util.Locale; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; -import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.common.breaker.CircuitBreaker; import org.sonar.api.utils.log.Loggers; import org.sonar.process.systeminfo.SystemInfoSection; import org.sonar.process.systeminfo.protobuf.ProtobufSystemInfo; import org.sonar.server.es.EsClient; +import org.sonar.server.es.response.NodeStats; +import org.sonar.server.es.response.NodeStatsResponse; import static java.lang.String.format; import static org.apache.commons.io.FileUtils.byteCountToDisplaySize; @@ -44,7 +43,7 @@ public class EsStateSection implements SystemInfoSection { } private ClusterHealthStatus getStateAsEnum() { - return clusterStats().getStatus(); + return esClient.clusterHealth(new ClusterHealthRequest()).getStatus(); } @Override @@ -54,7 +53,7 @@ public class EsStateSection implements SystemInfoSection { try { setAttribute(protobuf, "State", getStateAsEnum().name()); completeNodeAttributes(protobuf); - } catch (Exception es) { + } catch (Exception es) { Loggers.get(EsStateSection.class).warn("Failed to retrieve ES attributes. There will be only a single \"state\" attribute.", es); setAttribute(protobuf, "State", es.getCause() instanceof ElasticsearchException ? es.getCause().getMessage() : es.getMessage()); } @@ -62,42 +61,37 @@ public class EsStateSection implements SystemInfoSection { } private void completeNodeAttributes(ProtobufSystemInfo.Section.Builder protobuf) { - NodesStatsResponse nodesStats = esClient.prepareNodesStats() - .setFs(true) - .setProcess(true) - .setJvm(true) - .setIndices(true) - .setBreaker(true) - .get(); - if (!nodesStats.getNodes().isEmpty()) { - NodeStats stats = nodesStats.getNodes().get(0); - toProtobuf(stats, protobuf); + NodeStatsResponse nodesStatsResponse = esClient.nodesStats(); + + if (!nodesStatsResponse.getNodeStats().isEmpty()) { + toProtobuf(nodesStatsResponse.getNodeStats().get(0), protobuf); } } public static void toProtobuf(NodeStats stats, ProtobufSystemInfo.Section.Builder protobuf) { - setAttribute(protobuf, "CPU Usage (%)", stats.getProcess().getCpu().getPercent()); - setAttribute(protobuf, "Disk Available", byteCountToDisplaySize(stats.getFs().getTotal().getAvailable().getBytes())); - setAttribute(protobuf, "Store Size", byteCountToDisplaySize(stats.getIndices().getStore().getSizeInBytes())); - setAttribute(protobuf, "Translog Size", byteCountToDisplaySize(stats.getIndices().getTranslog().getTranslogSizeInBytes())); - setAttribute(protobuf, "Open File Descriptors", stats.getProcess().getOpenFileDescriptors()); - setAttribute(protobuf, "Max File Descriptors", stats.getProcess().getMaxFileDescriptors()); - setAttribute(protobuf, "JVM Heap Usage", formatPercent(stats.getJvm().getMem().getHeapUsedPercent())); - setAttribute(protobuf, "JVM Heap Used", byteCountToDisplaySize(stats.getJvm().getMem().getHeapUsed().getBytes())); - setAttribute(protobuf, "JVM Heap Max", byteCountToDisplaySize(stats.getJvm().getMem().getHeapMax().getBytes())); - setAttribute(protobuf, "JVM Non Heap Used", byteCountToDisplaySize(stats.getJvm().getMem().getNonHeapUsed().getBytes())); - setAttribute(protobuf, "JVM Threads", stats.getJvm().getThreads().getCount()); - setAttribute(protobuf, "Field Data Memory", byteCountToDisplaySize(stats.getIndices().getFieldData().getMemorySizeInBytes())); - setAttribute(protobuf, "Field Data Circuit Breaker Limit", byteCountToDisplaySize(stats.getBreaker().getStats(CircuitBreaker.FIELDDATA).getLimit())); - setAttribute(protobuf, "Field Data Circuit Breaker Estimation", byteCountToDisplaySize(stats.getBreaker().getStats(CircuitBreaker.FIELDDATA).getEstimated())); - setAttribute(protobuf, "Request Circuit Breaker Limit", byteCountToDisplaySize(stats.getBreaker().getStats(CircuitBreaker.REQUEST).getLimit())); - setAttribute(protobuf, "Request Circuit Breaker Estimation", byteCountToDisplaySize(stats.getBreaker().getStats(CircuitBreaker.REQUEST).getEstimated())); - setAttribute(protobuf, "Query Cache Memory", byteCountToDisplaySize(stats.getIndices().getQueryCache().getMemorySizeInBytes())); - setAttribute(protobuf, "Request Cache Memory", byteCountToDisplaySize(stats.getIndices().getRequestCache().getMemorySizeInBytes())); - } - - private ClusterStatsResponse clusterStats() { - return esClient.prepareClusterStats().get(); + setAttribute(protobuf, "CPU Usage (%)", stats.getCpuUsage()); + setAttribute(protobuf, "Disk Available", byteCountToDisplaySize(stats.getDiskAvailableBytes())); + setAttribute(protobuf, "Store Size", byteCountToDisplaySize(stats.getIndicesStats().getStoreSizeInBytes())); + setAttribute(protobuf, "Translog Size", byteCountToDisplaySize(stats.getIndicesStats().getTranslogSizeInBytes())); + setAttribute(protobuf, "Open File Descriptors", stats.getOpenFileDescriptors()); + setAttribute(protobuf, "Max File Descriptors", stats.getMaxFileDescriptors()); + setAttribute(protobuf, "JVM Heap Usage", formatPercent(stats.getJvmStats().getHeapUsedPercent())); + setAttribute(protobuf, "JVM Heap Used", byteCountToDisplaySize(stats.getJvmStats().getHeapUsedInBytes())); + setAttribute(protobuf, "JVM Heap Max", byteCountToDisplaySize(stats.getJvmStats().getHeapMaxInBytes())); + setAttribute(protobuf, "JVM Non Heap Used", byteCountToDisplaySize(stats.getJvmStats().getNonHeapUsedInBytes())); + setAttribute(protobuf, "JVM Threads", stats.getJvmStats().getThreadCount()); + setAttribute(protobuf, "Field Data Memory", byteCountToDisplaySize(stats.getIndicesStats().getFieldDataMemorySizeInBytes())); + setAttribute(protobuf, "Field Data Circuit Breaker Limit", + byteCountToDisplaySize(stats.getFieldDataCircuitBreakerLimit())); + setAttribute(protobuf, "Field Data Circuit Breaker Estimation", + byteCountToDisplaySize(stats.getFieldDataCircuitBreakerEstimation())); + setAttribute(protobuf, "Request Circuit Breaker Limit", + byteCountToDisplaySize(stats.getRequestCircuitBreakerLimit())); + setAttribute(protobuf, "Request Circuit Breaker Estimation", + byteCountToDisplaySize(stats.getRequestCircuitBreakerEstimation())); + setAttribute(protobuf, "Query Cache Memory", byteCountToDisplaySize(stats.getIndicesStats().getQueryCacheMemorySizeInBytes())); + setAttribute(protobuf, "Request Cache Memory", + byteCountToDisplaySize(stats.getIndicesStats().getRequestCacheMemorySizeInBytes())); } private static String formatPercent(long amount) { diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/cluster/EsClusterStateSection.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/cluster/EsClusterStateSection.java index 471893b03ee..e44efc3978e 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/cluster/EsClusterStateSection.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/cluster/EsClusterStateSection.java @@ -19,12 +19,12 @@ */ package org.sonar.server.platform.monitoring.cluster; -import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse; import org.sonar.api.server.ServerSide; import org.sonar.process.systeminfo.Global; import org.sonar.process.systeminfo.SystemInfoSection; import org.sonar.process.systeminfo.protobuf.ProtobufSystemInfo; import org.sonar.server.es.EsClient; +import org.sonar.server.es.response.ClusterStatsResponse; import static org.sonar.process.systeminfo.SystemInfoUtils.setAttribute; @@ -45,9 +45,9 @@ public class EsClusterStateSection implements SystemInfoSection, Global { public ProtobufSystemInfo.Section toProtobuf() { ProtobufSystemInfo.Section.Builder protobuf = ProtobufSystemInfo.Section.newBuilder(); protobuf.setName("Search State"); - ClusterStatsResponse stats = esClient.prepareClusterStats().get(); - setAttribute(protobuf, "State", stats.getStatus().name()); - setAttribute(protobuf, "Nodes", stats.getNodesStats().getCounts().getTotal()); + ClusterStatsResponse stats = esClient.clusterStats(); + setAttribute(protobuf, "State", stats.getHealthStatus().name()); + setAttribute(protobuf, "Nodes", stats.getNodeCount()); return protobuf.build(); } diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/cluster/SearchNodesInfoLoaderImpl.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/cluster/SearchNodesInfoLoaderImpl.java index 96cf66fc03a..ce0a0c062fd 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/cluster/SearchNodesInfoLoaderImpl.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/cluster/SearchNodesInfoLoaderImpl.java @@ -22,11 +22,11 @@ package org.sonar.server.platform.monitoring.cluster; import java.util.ArrayList; import java.util.Collection; import java.util.List; -import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.sonar.api.server.ServerSide; import org.sonar.process.systeminfo.protobuf.ProtobufSystemInfo; import org.sonar.server.es.EsClient; +import org.sonar.server.es.response.NodeStats; +import org.sonar.server.es.response.NodeStatsResponse; import org.sonar.server.platform.monitoring.EsStateSection; @ServerSide @@ -39,28 +39,22 @@ public class SearchNodesInfoLoaderImpl implements SearchNodesInfoLoader { } public Collection<NodeInfo> load() { - NodesStatsResponse nodesStats = esClient.prepareNodesStats() - .setFs(true) - .setProcess(true) - .setJvm(true) - .setIndices(true) - .setBreaker(true) - .get(); + NodeStatsResponse response = esClient.nodesStats(); + List<NodeInfo> result = new ArrayList<>(); - nodesStats.getNodes().forEach(nodeStat -> result.add(toNodeInfo(nodeStat))); + response.getNodeStats().forEach(nodeStat -> result.add(toNodeInfo(nodeStat))); return result; } private static NodeInfo toNodeInfo(NodeStats stat) { - String nodeName = stat.getNode().getName(); + String nodeName = stat.getName(); NodeInfo info = new NodeInfo(nodeName); - info.setHost(stat.getHostname()); + info.setHost(stat.getHost()); ProtobufSystemInfo.Section.Builder section = ProtobufSystemInfo.Section.newBuilder(); section.setName("Search State"); EsStateSection.toProtobuf(stat, section); info.addSection(section.build()); - return info; } diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java index 1746868bb25..f859000c2d2 100644 --- a/server/sonar-webserver-core/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java @@ -23,7 +23,8 @@ import com.google.common.collect.ImmutableMap; import java.util.Iterator; import java.util.Map; import javax.annotation.CheckForNull; -import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; +import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.junit.Rule; import org.junit.Test; @@ -33,6 +34,7 @@ import org.sonar.api.utils.log.LoggerLevel; import org.sonar.server.platform.db.migration.es.MigrationEsClient; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder; public class MigrationEsClientImplTest { @@ -77,8 +79,8 @@ public class MigrationEsClientImplTest { underTest.addMappingToExistingIndex("as", "s", "new_field", "keyword", mappingOptions); assertThat(loadExistingIndices()).toIterable().contains("as"); - ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = mappings(); - MappingMetaData mapping = mappings.get("as").get("s"); + ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetadata>> mappings = mappings(); + MappingMetadata mapping = mappings.get("as").get("s"); assertThat(countMappingFields(mapping)).isEqualTo(1); assertThat(field(mapping, "new_field")).isNotNull(); @@ -87,22 +89,31 @@ public class MigrationEsClientImplTest { assertThat(underTest.getUpdatedIndices()).containsExactly("as"); } + @Test + public void shouldFailIfMoreThanOneIndexReturned() { + String indexPattern = "*s"; + Map<String, String> mappingOptions = ImmutableMap.of("norms", "false"); + assertThatThrownBy(() -> underTest.addMappingToExistingIndex(indexPattern, "s", "new_field", "keyword", mappingOptions)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Expected only one index to be found, actual"); + } + private Iterator<String> loadExistingIndices() { - return es.client().nativeClient().admin().indices().prepareGetMappings().get().mappings().keysIt(); + return es.client().getMapping(new GetMappingsRequest()).mappings().keysIt(); } - private ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings() { - return es.client().nativeClient().admin().indices().prepareGetMappings().get().mappings(); + private ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetadata>> mappings() { + return es.client().getMapping(new GetMappingsRequest()).mappings(); } @CheckForNull @SuppressWarnings("unchecked") - private Map<String, Object> field(MappingMetaData mapping, String field) { + private Map<String, Object> field(MappingMetadata mapping, String field) { Map<String, Object> props = (Map<String, Object>) mapping.getSourceAsMap().get("properties"); return (Map<String, Object>) props.get(field); } - private int countMappingFields(MappingMetaData mapping) { + private int countMappingFields(MappingMetadata mapping) { return ((Map) mapping.getSourceAsMap().get("properties")).size(); } diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/monitoring/EsIndexesSectionTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/monitoring/EsIndexesSectionTest.java index e340ecf56c4..43559c7dbfe 100644 --- a/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/monitoring/EsIndexesSectionTest.java +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/monitoring/EsIndexesSectionTest.java @@ -27,6 +27,7 @@ import org.sonar.server.es.EsClient; import org.sonar.server.es.EsTester; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.sonar.process.systeminfo.SystemInfoUtils.attribute; @@ -58,7 +59,7 @@ public class EsIndexesSectionTest { public void attributes_displays_exception_message_when_cause_null_when_client_fails() { EsClient esClientMock = mock(EsClient.class); EsIndexesSection underTest = new EsIndexesSection(esClientMock); - when(esClientMock.prepareStats()).thenThrow(new RuntimeException("RuntimeException with no cause")); + when(esClientMock.indicesStats()).thenThrow(new RuntimeException("RuntimeException with no cause")); ProtobufSystemInfo.Section section = underTest.toProtobuf(); assertThatAttributeIs(section, "Error", "RuntimeException with no cause"); @@ -68,7 +69,7 @@ public class EsIndexesSectionTest { public void attributes_displays_exception_message_when_cause_is_not_ElasticSearchException_when_client_fails() { EsClient esClientMock = mock(EsClient.class); EsIndexesSection underTest = new EsIndexesSection(esClientMock); - when(esClientMock.prepareStats()).thenThrow(new RuntimeException("RuntimeException with cause not ES", new IllegalArgumentException("some cause message"))); + when(esClientMock.indicesStats()).thenThrow(new RuntimeException("RuntimeException with cause not ES", new IllegalArgumentException("some cause message"))); ProtobufSystemInfo.Section section = underTest.toProtobuf(); assertThatAttributeIs(section, "Error", "RuntimeException with cause not ES"); @@ -78,7 +79,7 @@ public class EsIndexesSectionTest { public void attributes_displays_cause_message_when_cause_is_ElasticSearchException_when_client_fails() { EsClient esClientMock = mock(EsClient.class); EsIndexesSection underTest = new EsIndexesSection(esClientMock); - when(esClientMock.prepareStats()).thenThrow(new RuntimeException("RuntimeException with ES cause", new ElasticsearchException("some cause message"))); + when(esClientMock.indicesStats()).thenThrow(new RuntimeException("RuntimeException with ES cause", new ElasticsearchException("some cause message"))); ProtobufSystemInfo.Section section = underTest.toProtobuf(); assertThatAttributeIs(section, "Error", "some cause message"); diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/monitoring/EsStateSectionTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/monitoring/EsStateSectionTest.java index f0445e2f831..d7da13bf907 100644 --- a/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/monitoring/EsStateSectionTest.java +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/monitoring/EsStateSectionTest.java @@ -28,6 +28,7 @@ import org.sonar.server.es.EsClient; import org.sonar.server.es.EsTester; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.sonar.process.systeminfo.SystemInfoUtils.attribute; @@ -63,7 +64,7 @@ public class EsStateSectionTest { public void attributes_displays_exception_message_when_cause_null_when_client_fails() { EsClient esClientMock = mock(EsClient.class); EsStateSection underTest = new EsStateSection(esClientMock); - when(esClientMock.prepareClusterStats()).thenThrow(new RuntimeException("RuntimeException with no cause")); + when(esClientMock.clusterHealth(any())).thenThrow(new RuntimeException("RuntimeException with no cause")); ProtobufSystemInfo.Section section = underTest.toProtobuf(); assertThatAttributeIs(section, "State", "RuntimeException with no cause"); @@ -73,7 +74,7 @@ public class EsStateSectionTest { public void attributes_displays_exception_message_when_cause_is_not_ElasticSearchException_when_client_fails() { EsClient esClientMock = mock(EsClient.class); EsStateSection underTest = new EsStateSection(esClientMock); - when(esClientMock.prepareClusterStats()).thenThrow(new RuntimeException("RuntimeException with cause not ES", new IllegalArgumentException("some cause message"))); + when(esClientMock.clusterHealth(any())).thenThrow(new RuntimeException("RuntimeException with cause not ES", new IllegalArgumentException("some cause message"))); ProtobufSystemInfo.Section section = underTest.toProtobuf(); assertThatAttributeIs(section, "State", "RuntimeException with cause not ES"); @@ -83,7 +84,7 @@ public class EsStateSectionTest { public void attributes_displays_cause_message_when_cause_is_ElasticSearchException_when_client_fails() { EsClient esClientMock = mock(EsClient.class); EsStateSection underTest = new EsStateSection(esClientMock); - when(esClientMock.prepareClusterStats()).thenThrow(new RuntimeException("RuntimeException with ES cause", new ElasticsearchException("some cause message"))); + when(esClientMock.clusterHealth(any())).thenThrow(new RuntimeException("RuntimeException with ES cause", new ElasticsearchException("some cause message"))); ProtobufSystemInfo.Section section = underTest.toProtobuf(); assertThatAttributeIs(section, "State", "some cause message"); diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/component/index/ComponentIndex.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/component/index/ComponentIndex.java index ad80fe587e0..489e18e758b 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/component/index/ComponentIndex.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/component/index/ComponentIndex.java @@ -26,7 +26,7 @@ import java.util.List; import java.util.function.Consumer; import java.util.stream.Stream; import javax.annotation.Nullable; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -35,10 +35,10 @@ import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregator.KeyedFilter; -import org.elasticsearch.search.aggregations.bucket.filter.InternalFilters; -import org.elasticsearch.search.aggregations.bucket.filter.InternalFilters.InternalBucket; -import org.elasticsearch.search.aggregations.metrics.tophits.InternalTopHits; -import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilters; +import org.elasticsearch.search.aggregations.metrics.ParsedTopHits; +import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.ScoreSortBuilder; @@ -81,11 +81,10 @@ public class ComponentIndex { } public SearchIdResult<String> search(ComponentQuery query, SearchOptions searchOptions) { - SearchRequestBuilder requestBuilder = client - .prepareSearch(TYPE_COMPONENT.getMainType()) - .setFetchSource(false) - .setFrom(searchOptions.getOffset()) - .setSize(searchOptions.getLimit()); + SearchSourceBuilder source = new SearchSourceBuilder() + .fetchSource(false) + .from(searchOptions.getOffset()) + .size(searchOptions.getLimit()); BoolQueryBuilder esQuery = boolQuery(); esQuery.filter(authorizationTypeSupport.createQueryFilter()); @@ -99,10 +98,13 @@ public class ComponentIndex { }); setEmptiable(query.getQualifiers(), q -> esQuery.must(termsQuery(FIELD_QUALIFIER, q))); setNullable(query.getOrganizationUuid(), o -> esQuery.must(termQuery(FIELD_ORGANIZATION_UUID, o))); - requestBuilder.setQuery(esQuery); - requestBuilder.addSort(SORTABLE_ANALYZER.subField(FIELD_NAME), SortOrder.ASC); + source.sort(SORTABLE_ANALYZER.subField(FIELD_NAME), SortOrder.ASC); - return new SearchIdResult<>(requestBuilder.get(), id -> id, system2.getDefaultTimeZone()); + source.query(esQuery); + + SearchRequest request = EsClient.prepareSearch(TYPE_COMPONENT.getMainType()) + .source(source); + return new SearchIdResult<>(client.search(request), id -> id, system2.getDefaultTimeZone()); } public ComponentIndexResults searchSuggestions(SuggestionQuery query) { @@ -116,15 +118,16 @@ public class ComponentIndex { return ComponentIndexResults.newBuilder().build(); } - SearchRequestBuilder request = client - .prepareSearch(TYPE_COMPONENT.getMainType()) - .setQuery(createQuery(query, features)) - .addAggregation(createAggregation(query)) + SearchSourceBuilder source = new SearchSourceBuilder() + .query(createQuery(query, features)) + .aggregation(createAggregation(query)) // the search hits are part of the aggregations - .setSize(0); + .size(0); - SearchResponse response = request.get(); + SearchRequest request = EsClient.prepareSearch(TYPE_COMPONENT.getMainType()) + .source(source); + SearchResponse response = client.search(request); return aggregationsToQualifiers(response); } @@ -178,21 +181,21 @@ public class ComponentIndex { } private static ComponentIndexResults aggregationsToQualifiers(SearchResponse response) { - InternalFilters filtersAgg = response.getAggregations().get(FILTERS_AGGREGATION_NAME); - List<InternalBucket> buckets = filtersAgg.getBuckets(); + ParsedFilters filtersAgg = response.getAggregations().get(FILTERS_AGGREGATION_NAME); + List<ParsedFilters.ParsedBucket> buckets = (List<ParsedFilters.ParsedBucket>) filtersAgg.getBuckets(); return ComponentIndexResults.newBuilder() .setQualifiers( buckets.stream().map(ComponentIndex::bucketToQualifier)) .build(); } - private static ComponentHitsPerQualifier bucketToQualifier(InternalBucket bucket) { - InternalTopHits docs = bucket.getAggregations().get(DOCS_AGGREGATION_NAME); + private static ComponentHitsPerQualifier bucketToQualifier(ParsedFilters.ParsedBucket bucket) { + ParsedTopHits docs = bucket.getAggregations().get(DOCS_AGGREGATION_NAME); SearchHits hitList = docs.getHits(); SearchHit[] hits = hitList.getHits(); - return new ComponentHitsPerQualifier(bucket.getKey(), ComponentHit.fromSearchHits(hits), hitList.getTotalHits()); + return new ComponentHitsPerQualifier(bucket.getKey(), ComponentHit.fromSearchHits(hits), hitList.getTotalHits().value); } private static <T> void setNullable(@Nullable T parameter, Consumer<T> consumer) { diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/es/IndexCreator.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/es/IndexCreator.java index e671e14a38f..9f16edab51c 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/es/IndexCreator.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/es/IndexCreator.java @@ -25,10 +25,14 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.client.indices.CreateIndexResponse; +import org.elasticsearch.client.indices.GetIndexRequest; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.settings.Settings; import org.picocontainer.Startable; @@ -78,7 +82,7 @@ public class IndexCreator implements Startable { public void start() { // create the "metadata" index first IndexType.IndexMainType metadataMainType = TYPE_METADATA; - if (!client.prepareIndicesExist(metadataMainType.getIndex()).get().isExists()) { + if (!client.indexExists(new GetIndexRequest(metadataMainType.getIndex().getName()))) { IndexDefinition.IndexDefinitionContext context = new IndexDefinition.IndexDefinitionContext(); metadataIndexDefinition.define(context); NewIndex index = context.getIndices().values().iterator().next(); @@ -93,7 +97,7 @@ public class IndexCreator implements Startable { definitions.getIndices().values().stream() .filter(i -> !i.getMainType().equals(metadataMainType)) .forEach(index -> { - boolean exists = client.prepareIndicesExist(index.getMainType().getIndex()).get().isExists(); + boolean exists = client.indexExists(new GetIndexRequest(index.getMainType().getIndex().getName())); if (!exists) { createIndex(index, true); } else if (hasDefinitionChange(index)) { @@ -112,7 +116,7 @@ public class IndexCreator implements Startable { private boolean isReadOnly(IndexType.IndexMainType mainType) { String indexName = mainType.getIndex().getName(); - String readOnly = client.nativeClient().admin().indices().getSettings(new GetSettingsRequest().indices(indexName)).actionGet() + String readOnly = client.getSettings(new GetSettingsRequest().indices(indexName)) .getSetting(indexName, "index.blocks.read_only_allow_delete"); return "true".equalsIgnoreCase(readOnly); } @@ -123,9 +127,8 @@ public class IndexCreator implements Startable { String indexName = mainType.getIndex().getName(); Settings.Builder builder = Settings.builder(); builder.putNull("index.blocks.read_only_allow_delete"); - client.nativeClient().admin().indices() - .updateSettings(new UpdateSettingsRequest().indices(indexName).settings(builder.build())) - .actionGet(); + + client.putSettings(new UpdateSettingsRequest().indices(indexName).settings(builder.build())); } @Override @@ -143,10 +146,8 @@ public class IndexCreator implements Startable { metadataIndex.setInitialized(builtIndex.getMainType(), false); builtIndex.getRelationTypes().forEach(relationType -> metadataIndex.setInitialized(relationType, false)); } - CreateIndexResponse indexResponse = client - .prepareCreate(index) - .setSettings(settings) - .get(); + CreateIndexResponse indexResponse = client.create(new CreateIndexRequest(index.getName()).settings((settings))); + if (!indexResponse.isAcknowledged()) { throw new IllegalStateException("Failed to create index [" + index.getName() + "]"); } @@ -154,10 +155,10 @@ public class IndexCreator implements Startable { // create types LOGGER.info("Create type {}", builtIndex.getMainType().format()); - AcknowledgedResponse mappingResponse = client.preparePutMapping(index) - .setType(builtIndex.getMainType().getType()) - .setSource(builtIndex.getAttributes()) - .get(); + AcknowledgedResponse mappingResponse = client.putMapping(new PutMappingRequest(builtIndex.getMainType().getIndex().getName()) + .type(builtIndex.getMainType().getType()) + .source(builtIndex.getAttributes())); + if (!mappingResponse.isAcknowledged()) { throw new IllegalStateException("Failed to create type " + builtIndex.getMainType().getType()); } @@ -165,7 +166,7 @@ public class IndexCreator implements Startable { } private void deleteIndex(String indexName) { - client.nativeClient().admin().indices().prepareDelete(indexName).get(); + client.deleteIndex(new DeleteIndexRequest(indexName)); } private void updateIndex(BuiltIndex<?> index) { @@ -215,7 +216,7 @@ public class IndexCreator implements Startable { Set<String> definedNames = definitions.stream() .map(t -> t.getMainType().getIndex().getName()) .collect(Collectors.toSet()); - return Arrays.stream(client.nativeClient().admin().indices().prepareGetIndex().get().getIndices()) + return Arrays.stream(client.getIndex(new GetIndexRequest("_all")).getIndices()) .filter(definedNames::contains) .filter(index -> !DESCRIPTOR.getName().equals(index)) .collect(Collectors.toList()); diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java index 9afa4787e17..1446e48f4ab 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java @@ -38,7 +38,7 @@ import java.util.stream.Stream; import javax.annotation.CheckForNull; import javax.annotation.Nullable; import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -48,21 +48,21 @@ import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.HasAggregations; +import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.filter.InternalFilter; +import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilter; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds; import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude; -import org.elasticsearch.search.aggregations.bucket.terms.InternalTerms; -import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; +import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.max.InternalMax; -import org.elasticsearch.search.aggregations.metrics.min.Min; -import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.valuecount.InternalValueCount; +import org.elasticsearch.search.aggregations.metrics.Min; +import org.elasticsearch.search.aggregations.metrics.ParsedMax; +import org.elasticsearch.search.aggregations.metrics.ParsedValueCount; +import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; -import org.joda.time.DateTimeZone; import org.joda.time.Duration; import org.sonar.api.issue.Issue; import org.sonar.api.rule.Severity; @@ -88,6 +88,7 @@ import org.sonar.server.issue.index.IssueQuery.PeriodStart; import org.sonar.server.permission.index.AuthorizationDoc; import org.sonar.server.permission.index.WebAuthorizationTypeSupport; import org.sonar.server.security.SecurityStandards; +import org.sonar.server.security.SecurityStandards.SQCategory; import org.sonar.server.user.UserSession; import org.sonar.server.view.index.ViewIndexDefinition; @@ -344,39 +345,41 @@ public class IssueIndex { } public SearchResponse search(IssueQuery query, SearchOptions options) { - SearchRequestBuilder esRequest = client.prepareSearch(TYPE_ISSUE.getMainType()); + SearchRequest requestBuilder = EsClient.prepareSearch(TYPE_ISSUE.getMainType()); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + requestBuilder.source(sourceBuilder); - configureSorting(query, esRequest); - configurePagination(options, esRequest); - configureRouting(query, options, esRequest); + configureSorting(query, sourceBuilder); + configurePagination(options, sourceBuilder); + configureRouting(query, options, requestBuilder); AllFilters allFilters = createAllFilters(query); RequestFiltersComputer filterComputer = newFilterComputer(options, allFilters); - configureTopAggregations(query, options, esRequest, allFilters, filterComputer); - configureQuery(esRequest, filterComputer); - configureTopFilters(esRequest, filterComputer); + configureTopAggregations(query, options, sourceBuilder, allFilters, filterComputer); + configureQuery(sourceBuilder, filterComputer); + configureTopFilters(sourceBuilder, filterComputer); - esRequest.setFetchSource(false); + sourceBuilder.fetchSource(false); - return esRequest.get(); + return client.search(requestBuilder); } - private void configureTopAggregations(IssueQuery query, SearchOptions options, SearchRequestBuilder esRequest, AllFilters allFilters, RequestFiltersComputer filterComputer) { + private void configureTopAggregations(IssueQuery query, SearchOptions options, SearchSourceBuilder esRequest, AllFilters allFilters, RequestFiltersComputer filterComputer) { TopAggregationHelper aggregationHelper = newAggregationHelper(filterComputer, query); configureTopAggregations(aggregationHelper, query, options, allFilters, esRequest); } - private static void configureQuery(SearchRequestBuilder esRequest, RequestFiltersComputer filterComputer) { + private static void configureQuery(SearchSourceBuilder esRequest, RequestFiltersComputer filterComputer) { QueryBuilder esQuery = filterComputer.getQueryFilters() .map(t -> (QueryBuilder) boolQuery().must(matchAllQuery()).filter(t)) .orElse(matchAllQuery()); - esRequest.setQuery(esQuery); + esRequest.query(esQuery); } - private static void configureTopFilters(SearchRequestBuilder esRequest, RequestFiltersComputer filterComputer) { - filterComputer.getPostFilters().ifPresent(esRequest::setPostFilter); + private static void configureTopFilters(SearchSourceBuilder esRequest, RequestFiltersComputer filterComputer) { + filterComputer.getPostFilters().ifPresent(esRequest::postFilter); } /** @@ -386,15 +389,15 @@ public class IssueIndex { * Note that sticky facets may involve all projects, so this optimization must be * disabled when facets are enabled. */ - private static void configureRouting(IssueQuery query, SearchOptions options, SearchRequestBuilder requestBuilder) { + private static void configureRouting(IssueQuery query, SearchOptions options, SearchRequest searchRequest) { Collection<String> uuids = query.projectUuids(); if (!uuids.isEmpty() && options.getFacets().isEmpty()) { - requestBuilder.setRouting(uuids.stream().map(AuthorizationDoc::idOf).toArray(String[]::new)); + searchRequest.routing(uuids.stream().map(AuthorizationDoc::idOf).toArray(String[]::new)); } } - private static void configurePagination(SearchOptions options, SearchRequestBuilder esSearch) { - esSearch.setFrom(options.getOffset()).setSize(options.getLimit()); + private static void configurePagination(SearchOptions options, SearchSourceBuilder esSearch) { + esSearch.from(options.getOffset()).size(options.getLimit()); } private AllFilters createAllFilters(IssueQuery query) { @@ -596,8 +599,8 @@ public class IssueIndex { return value == null ? null : termQuery(field, value); } - private void configureSorting(IssueQuery query, SearchRequestBuilder esRequest) { - createSortBuilders(query).forEach(esRequest::addSort); + private void configureSorting(IssueQuery query, SearchSourceBuilder esRequest) { + createSortBuilders(query).forEach(esRequest::sort); } private List<FieldSortBuilder> createSortBuilders(IssueQuery query) { @@ -659,7 +662,7 @@ public class IssueIndex { } private void configureTopAggregations(TopAggregationHelper aggregationHelper, IssueQuery query, SearchOptions options, - AllFilters queryFilters, SearchRequestBuilder esRequest) { + AllFilters queryFilters, SearchSourceBuilder esRequest) { addFacetIfNeeded(options, aggregationHelper, esRequest, STATUSES, NO_SELECTED_VALUES); addFacetIfNeeded(options, aggregationHelper, esRequest, PROJECT_UUIDS, query.projectUuids().toArray()); addFacetIfNeeded(options, aggregationHelper, esRequest, MODULE_UUIDS, query.moduleUuids().toArray()); @@ -687,7 +690,7 @@ public class IssueIndex { } private static void addFacetIfNeeded(SearchOptions options, TopAggregationHelper aggregationHelper, - SearchRequestBuilder esRequest, Facet facet, Object[] selectedValues) { + SearchSourceBuilder esRequest, Facet facet, Object[] selectedValues) { if (!options.getFacets().contains(facet.getName())) { return; } @@ -697,10 +700,10 @@ public class IssueIndex { NO_EXTRA_FILTER, t -> aggregationHelper.getSubAggregationHelper().buildSelectedItemsAggregation(facet.getName(), facet.getTopAggregationDef(), selectedValues) .ifPresent(t::subAggregation)); - esRequest.addAggregation(topAggregation); + esRequest.aggregation(topAggregation); } - private static void addSecurityCategoryFacetIfNeeded(String param, Facet facet, SearchOptions options, TopAggregationHelper aggregationHelper, SearchRequestBuilder esRequest, + private static void addSecurityCategoryFacetIfNeeded(String param, Facet facet, SearchOptions options, TopAggregationHelper aggregationHelper, SearchSourceBuilder esRequest, Object[] selectedValues) { if (!options.getFacets().contains(param)) { return; @@ -711,10 +714,10 @@ public class IssueIndex { filter -> filter.must(termQuery(FIELD_ISSUE_TYPE, VULNERABILITY.name())), t -> aggregationHelper.getSubAggregationHelper().buildSelectedItemsAggregation(facet.getName(), facet.getTopAggregationDef(), selectedValues) .ifPresent(t::subAggregation)); - esRequest.addAggregation(aggregation); + esRequest.aggregation(aggregation); } - private static void addSeverityFacetIfNeeded(SearchOptions options, TopAggregationHelper aggregationHelper, SearchRequestBuilder esRequest) { + private static void addSeverityFacetIfNeeded(SearchOptions options, TopAggregationHelper aggregationHelper, SearchSourceBuilder esRequest) { if (!options.getFacets().contains(PARAM_SEVERITIES)) { return; } @@ -724,10 +727,10 @@ public class IssueIndex { // Ignore severity of Security HotSpots filter -> filter.mustNot(termQuery(FIELD_ISSUE_TYPE, SECURITY_HOTSPOT.name())), NO_OTHER_SUBAGGREGATION); - esRequest.addAggregation(aggregation); + esRequest.aggregation(aggregation); } - private static void addResolutionFacetIfNeeded(SearchOptions options, IssueQuery query, TopAggregationHelper aggregationHelper, SearchRequestBuilder esRequest) { + private static void addResolutionFacetIfNeeded(SearchOptions options, IssueQuery query, TopAggregationHelper aggregationHelper, SearchSourceBuilder esRequest) { if (!options.getFacets().contains(PARAM_RESOLUTIONS)) { return; } @@ -742,10 +745,10 @@ public class IssueIndex { .missing(RESOLUTIONS.getName() + FACET_SUFFIX_MISSING) .field(RESOLUTIONS.getFieldName()))); }); - esRequest.addAggregation(aggregation); + esRequest.aggregation(aggregation); } - private static void addAssigneesFacetIfNeeded(SearchOptions options, IssueQuery query, TopAggregationHelper aggregationHelper, SearchRequestBuilder esRequest) { + private static void addAssigneesFacetIfNeeded(SearchOptions options, IssueQuery query, TopAggregationHelper aggregationHelper, SearchSourceBuilder esRequest) { if (!options.getFacets().contains(PARAM_ASSIGNEES)) { return; } @@ -765,13 +768,13 @@ public class IssueIndex { AggregationBuilder aggregation = aggregationHelper.buildTermTopAggregation( ASSIGNEES.getName(), ASSIGNEES.getTopAggregationDef(), ASSIGNEES.getNumberOfTerms(), NO_EXTRA_FILTER, assigneeAggregations); - esRequest.addAggregation(aggregation); + esRequest.aggregation(aggregation); } private void addCreatedAtFacetIfNeeded(SearchOptions options, IssueQuery query, TopAggregationHelper aggregationHelper, AllFilters allFilters, - SearchRequestBuilder esRequest) { + SearchSourceBuilder esRequest) { if (options.getFacets().contains(PARAM_CREATED_AT)) { - getCreatedAtFacet(query, aggregationHelper, allFilters).ifPresent(esRequest::addAggregation); + getCreatedAtFacet(query, aggregationHelper, allFilters).ifPresent(esRequest::aggregation); } } @@ -806,7 +809,7 @@ public class IssueIndex { .dateHistogramInterval(bucketSize) .minDocCount(0L) .format(DateUtils.DATETIME_FORMAT) - .timeZone(DateTimeZone.forOffsetMillis(system.getDefaultTimeZone().getRawOffset())) + .timeZone(system.getDefaultTimeZone().toZoneId()) // ES dateHistogram bounds are inclusive while createdBefore parameter is exclusive .extendedBounds(new ExtendedBounds(startInclusive ? startTime : (startTime + 1), endTime - 1L)); addEffortAggregationIfNeeded(query, dateHistogram); @@ -831,17 +834,20 @@ public class IssueIndex { private OptionalLong getMinCreatedAt(AllFilters filters) { String facetNameAndField = CREATED_AT.getFieldName(); - SearchRequestBuilder esRequest = client - .prepareSearch(TYPE_ISSUE.getMainType()) - .setSize(0); + + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() + .size(0); BoolQueryBuilder esFilter = boolQuery(); filters.stream().filter(Objects::nonNull).forEach(esFilter::must); if (esFilter.hasClauses()) { - esRequest.setQuery(QueryBuilders.boolQuery().filter(esFilter)); + sourceBuilder.query(QueryBuilders.boolQuery().filter(esFilter)); } - esRequest.addAggregation(AggregationBuilders.min(facetNameAndField).field(facetNameAndField)); - Min minValue = esRequest.get().getAggregations().get(facetNameAndField); + sourceBuilder.aggregation(AggregationBuilders.min(facetNameAndField).field(facetNameAndField)); + + SearchRequest request = EsClient.prepareSearch(TYPE_ISSUE.getMainType()) + .source(sourceBuilder); + Min minValue = client.search(request).getAggregations().get(facetNameAndField); double actualValue = minValue.getValue(); if (Double.isInfinite(actualValue)) { return OptionalLong.empty(); @@ -849,7 +855,7 @@ public class IssueIndex { return OptionalLong.of((long) actualValue); } - private void addAssignedToMeFacetIfNeeded(SearchOptions options, TopAggregationHelper aggregationHelper, SearchRequestBuilder esRequest) { + private void addAssignedToMeFacetIfNeeded(SearchOptions options, TopAggregationHelper aggregationHelper, SearchSourceBuilder esRequest) { String uuid = userSession.getUuid(); if (options.getFacets().contains(ASSIGNED_TO_ME.getName()) && !StringUtils.isEmpty(uuid)) { AggregationBuilder aggregation = aggregationHelper.buildTermTopAggregation( @@ -863,17 +869,17 @@ public class IssueIndex { .buildSelectedItemsAggregation(ASSIGNED_TO_ME.getName(), ASSIGNED_TO_ME.getTopAggregationDef(), new String[] {uuid}) .ifPresent(t::subAggregation); }); - esRequest.addAggregation(aggregation); + esRequest.aggregation(aggregation); } } - private static void addEffortTopAggregation(TopAggregationHelper aggregationHelper, SearchRequestBuilder esRequest) { + private static void addEffortTopAggregation(TopAggregationHelper aggregationHelper, SearchSourceBuilder esRequest) { AggregationBuilder topAggregation = aggregationHelper.buildTopAggregation( FACET_MODE_EFFORT, EFFORT_TOP_AGGREGATION, NO_EXTRA_FILTER, t -> t.subAggregation(EFFORT_AGGREGATION)); - esRequest.addAggregation(topAggregation); + esRequest.aggregation(topAggregation); } public List<String> searchTags(IssueQuery query, @Nullable String textQuery, int size) { @@ -892,12 +898,14 @@ public class IssueIndex { } private Terms listTermsMatching(String fieldName, IssueQuery query, @Nullable String textQuery, BucketOrder termsOrder, int size) { - SearchRequestBuilder requestBuilder = client - .prepareSearch(TYPE_ISSUE.getMainType()) + SearchRequest requestBuilder = EsClient.prepareSearch(TYPE_ISSUE.getMainType()); + + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() // Avoids returning search hits - .setSize(0); + .size(0); + requestBuilder.source(sourceBuilder); - requestBuilder.setQuery(boolQuery().must(QueryBuilders.matchAllQuery()).filter(createBoolFilter(query))); + sourceBuilder.query(boolQuery().must(QueryBuilders.matchAllQuery()).filter(createBoolFilter(query))); TermsAggregationBuilder aggreg = AggregationBuilders.terms("_ref") .field(fieldName) @@ -908,7 +916,9 @@ public class IssueIndex { aggreg.includeExclude(new IncludeExclude(format(SUBSTRING_MATCH_REGEXP, escapeSpecialRegexChars(textQuery)), null)); } - SearchResponse searchResponse = requestBuilder.addAggregation(aggreg).get(); + sourceBuilder.aggregation(aggreg); + + SearchResponse searchResponse = client.search(requestBuilder); return searchResponse.getAggregations().get("_ref"); } @@ -926,18 +936,19 @@ public class IssueIndex { if (projectUuids.isEmpty()) { return Collections.emptyList(); } - SearchRequestBuilder request = client.prepareSearch(TYPE_ISSUE.getMainType()) - .setQuery( + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() + .query( boolQuery() .mustNot(existsQuery(FIELD_ISSUE_RESOLUTION)) .filter(termQuery(FIELD_ISSUE_ASSIGNEE_UUID, assigneeUuid)) .mustNot(termQuery(FIELD_ISSUE_TYPE, SECURITY_HOTSPOT.name()))) - .setSize(0); + .size(0); + IntStream.range(0, projectUuids.size()).forEach(i -> { String projectUuid = projectUuids.get(i); long from = froms.get(i); - request - .addAggregation(AggregationBuilders + sourceBuilder + .aggregation(AggregationBuilders .filter(projectUuid, boolQuery() .filter(termQuery(FIELD_ISSUE_PROJECT_UUID, projectUuid)) .filter(rangeQuery(FIELD_ISSUE_FUNC_CREATED_AT).gte(epochMillisToEpochSeconds(from)))) @@ -948,16 +959,20 @@ public class IssueIndex { .subAggregation( AggregationBuilders.max("maxFuncCreatedAt").field(FIELD_ISSUE_FUNC_CREATED_AT)))); }); - SearchResponse response = request.get(); + + SearchRequest requestBuilder = EsClient.prepareSearch(TYPE_ISSUE.getMainType()); + + requestBuilder.source(sourceBuilder); + SearchResponse response = client.search(requestBuilder); return response.getAggregations().asList().stream() - .map(x -> (InternalFilter) x) - .flatMap(projectBucket -> ((StringTerms) projectBucket.getAggregations().get("branchUuid")).getBuckets().stream() + .map(x -> (ParsedFilter) x) + .flatMap(projectBucket -> ((ParsedStringTerms) projectBucket.getAggregations().get("branchUuid")).getBuckets().stream() .flatMap(branchBucket -> { - long count = ((InternalValueCount) branchBucket.getAggregations().get(AGG_COUNT)).getValue(); + long count = ((ParsedValueCount) branchBucket.getAggregations().get(AGG_COUNT)).getValue(); if (count < 1L) { return Stream.empty(); } - long lastIssueDate = (long) ((InternalMax) branchBucket.getAggregations().get("maxFuncCreatedAt")).getValue(); + long lastIssueDate = (long) ((ParsedMax) branchBucket.getAggregations().get("maxFuncCreatedAt")).getValue(); return Stream.of(new ProjectStatistics(branchBucket.getKeyAsString(), count, lastIssueDate)); })) .collect(MoreCollectors.toList(projectUuids.size())); @@ -968,32 +983,36 @@ public class IssueIndex { return Collections.emptyList(); } - SearchRequestBuilder request = client.prepareSearch(TYPE_ISSUE.getMainType()) - .setRouting(AuthorizationDoc.idOf(projectUuid)) - .setQuery( + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() + .query( boolQuery() .must(termsQuery(FIELD_ISSUE_BRANCH_UUID, branchUuids)) .mustNot(existsQuery(FIELD_ISSUE_RESOLUTION)) .must(termQuery(FIELD_ISSUE_IS_MAIN_BRANCH, Boolean.toString(false)))) - .setSize(0) - .addAggregation(AggregationBuilders.terms("branchUuids") + .size(0) + .aggregation(AggregationBuilders.terms("branchUuids") .field(FIELD_ISSUE_BRANCH_UUID) .size(branchUuids.size()) .subAggregation(AggregationBuilders.terms("types") .field(FIELD_ISSUE_TYPE))); - SearchResponse response = request.get(); - return ((StringTerms) response.getAggregations().get("branchUuids")).getBuckets().stream() + + SearchRequest requestBuilder = EsClient.prepareSearch(TYPE_ISSUE.getMainType()) + .routing(AuthorizationDoc.idOf(projectUuid)); + + requestBuilder.source(sourceBuilder); + SearchResponse response = client.search(requestBuilder); + return ((ParsedStringTerms) response.getAggregations().get("branchUuids")).getBuckets().stream() .map(bucket -> new PrStatistics(bucket.getKeyAsString(), - ((StringTerms) bucket.getAggregations().get("types")).getBuckets() + ((ParsedStringTerms) bucket.getAggregations().get("types")).getBuckets() .stream() - .collect(uniqueIndex(StringTerms.Bucket::getKeyAsString, InternalTerms.Bucket::getDocCount)))) + .collect(uniqueIndex(MultiBucketsAggregation.Bucket::getKeyAsString, MultiBucketsAggregation.Bucket::getDocCount)))) .collect(MoreCollectors.toList(branchUuids.size())); } public List<SecurityStandardCategoryStatistics> getSansTop25Report(String projectUuid, boolean isViewOrApp, boolean includeCwe) { - SearchRequestBuilder request = prepareNonClosedVulnerabilitiesAndHotspotSearch(projectUuid, isViewOrApp); + SearchSourceBuilder request = prepareNonClosedVulnerabilitiesAndHotspotSearch(projectUuid, isViewOrApp); Stream.of(SANS_TOP_25_INSECURE_INTERACTION, SANS_TOP_25_RISKY_RESOURCE, SANS_TOP_25_POROUS_DEFENSES) - .forEach(sansCategory -> request.addAggregation(newSecurityReportSubAggregations( + .forEach(sansCategory -> request.aggregation(newSecurityReportSubAggregations( AggregationBuilders.filter(sansCategory, boolQuery().filter(termQuery(FIELD_ISSUE_SANS_TOP_25, sansCategory))), includeCwe, Optional.ofNullable(SecurityStandards.CWES_BY_SANS_TOP_25.get(sansCategory))))); @@ -1001,9 +1020,9 @@ public class IssueIndex { } public List<SecurityStandardCategoryStatistics> getSonarSourceReport(String projectUuid, boolean isViewOrApp, boolean includeCwe) { - SearchRequestBuilder request = prepareNonClosedVulnerabilitiesAndHotspotSearch(projectUuid, isViewOrApp); - Arrays.stream(SecurityStandards.SQCategory.values()) - .forEach(sonarsourceCategory -> request.addAggregation( + SearchSourceBuilder request = prepareNonClosedVulnerabilitiesAndHotspotSearch(projectUuid, isViewOrApp); + Arrays.stream(SQCategory.values()) + .forEach(sonarsourceCategory -> request.aggregation( newSecurityReportSubAggregations( AggregationBuilders.filter(sonarsourceCategory.getKey(), boolQuery().filter(termQuery(FIELD_ISSUE_SQ_SECURITY_CATEGORY, sonarsourceCategory.getKey()))), includeCwe, @@ -1012,9 +1031,9 @@ public class IssueIndex { } public List<SecurityStandardCategoryStatistics> getOwaspTop10Report(String projectUuid, boolean isViewOrApp, boolean includeCwe) { - SearchRequestBuilder request = prepareNonClosedVulnerabilitiesAndHotspotSearch(projectUuid, isViewOrApp); + SearchSourceBuilder request = prepareNonClosedVulnerabilitiesAndHotspotSearch(projectUuid, isViewOrApp); IntStream.rangeClosed(1, 10).mapToObj(i -> "a" + i) - .forEach(owaspCategory -> request.addAggregation( + .forEach(owaspCategory -> request.aggregation( newSecurityReportSubAggregations( AggregationBuilders.filter(owaspCategory, boolQuery().filter(termQuery(FIELD_ISSUE_OWASP_TOP_10, owaspCategory))), includeCwe, @@ -1022,17 +1041,19 @@ public class IssueIndex { return processSecurityReportSearchResults(request, includeCwe); } - private static List<SecurityStandardCategoryStatistics> processSecurityReportSearchResults(SearchRequestBuilder request, boolean includeCwe) { - SearchResponse response = request.get(); + private List<SecurityStandardCategoryStatistics> processSecurityReportSearchResults(SearchSourceBuilder sourceBuilder, boolean includeCwe) { + SearchRequest request = EsClient.prepareSearch(TYPE_ISSUE.getMainType()) + .source(sourceBuilder); + SearchResponse response = client.search(request); return response.getAggregations().asList().stream() - .map(c -> processSecurityReportIssueSearchResults((InternalFilter) c, includeCwe)) + .map(c -> processSecurityReportIssueSearchResults((ParsedFilter) c, includeCwe)) .collect(MoreCollectors.toList()); } - private static SecurityStandardCategoryStatistics processSecurityReportIssueSearchResults(InternalFilter categoryBucket, boolean includeCwe) { + private static SecurityStandardCategoryStatistics processSecurityReportIssueSearchResults(ParsedFilter categoryBucket, boolean includeCwe) { List<SecurityStandardCategoryStatistics> children = new ArrayList<>(); if (includeCwe) { - Stream<StringTerms.Bucket> stream = ((StringTerms) categoryBucket.getAggregations().get(AGG_CWES)).getBuckets().stream(); + Stream<? extends Terms.Bucket> stream = ((ParsedStringTerms) categoryBucket.getAggregations().get(AGG_CWES)).getBuckets().stream(); children = stream.map(cweBucket -> processSecurityReportCategorySearchResults(cweBucket, cweBucket.getKeyAsString(), null)).collect(toList()); } @@ -1041,18 +1062,18 @@ public class IssueIndex { private static SecurityStandardCategoryStatistics processSecurityReportCategorySearchResults(HasAggregations categoryBucket, String categoryName, @Nullable List<SecurityStandardCategoryStatistics> children) { - List<StringTerms.Bucket> severityBuckets = ((StringTerms) ((InternalFilter) categoryBucket.getAggregations().get(AGG_VULNERABILITIES)).getAggregations().get(AGG_SEVERITIES)) - .getBuckets(); - long vulnerabilities = severityBuckets.stream().mapToLong(b -> ((InternalValueCount) b.getAggregations().get(AGG_COUNT)).getValue()).sum(); + List<? extends Terms.Bucket> severityBuckets = ((ParsedStringTerms) ((ParsedFilter) categoryBucket.getAggregations().get(AGG_VULNERABILITIES)).getAggregations() + .get(AGG_SEVERITIES)).getBuckets(); + long vulnerabilities = severityBuckets.stream().mapToLong(b -> ((ParsedValueCount) b.getAggregations().get(AGG_COUNT)).getValue()).sum(); // Worst severity having at least one issue OptionalInt severityRating = severityBuckets.stream() - .filter(b -> ((InternalValueCount) b.getAggregations().get(AGG_COUNT)).getValue() != 0) + .filter(b -> ((ParsedValueCount) b.getAggregations().get(AGG_COUNT)).getValue() != 0) .mapToInt(b -> Severity.ALL.indexOf(b.getKeyAsString()) + 1) .max(); - long toReviewSecurityHotspots = ((InternalValueCount) ((InternalFilter) categoryBucket.getAggregations().get(AGG_TO_REVIEW_SECURITY_HOTSPOTS)).getAggregations().get(AGG_COUNT)) + long toReviewSecurityHotspots = ((ParsedValueCount) ((ParsedFilter) categoryBucket.getAggregations().get(AGG_TO_REVIEW_SECURITY_HOTSPOTS)).getAggregations().get(AGG_COUNT)) .getValue(); - long reviewedSecurityHotspots = ((InternalValueCount) ((InternalFilter) categoryBucket.getAggregations().get(AGG_REVIEWED_SECURITY_HOTSPOTS)).getAggregations().get(AGG_COUNT)) + long reviewedSecurityHotspots = ((ParsedValueCount) ((ParsedFilter) categoryBucket.getAggregations().get(AGG_REVIEWED_SECURITY_HOTSPOTS)).getAggregations().get(AGG_COUNT)) .getValue(); Optional<Double> percent = computePercent(toReviewSecurityHotspots, reviewedSecurityHotspots); @@ -1096,7 +1117,7 @@ public class IssueIndex { AggregationBuilders.count(AGG_COUNT).field(FIELD_ISSUE_KEY))); } - private SearchRequestBuilder prepareNonClosedVulnerabilitiesAndHotspotSearch(String projectUuid, boolean isViewOrApp) { + private static SearchSourceBuilder prepareNonClosedVulnerabilitiesAndHotspotSearch(String projectUuid, boolean isViewOrApp) { BoolQueryBuilder componentFilter = boolQuery(); if (isViewOrApp) { IndexType.IndexMainType mainType = TYPE_VIEW; @@ -1109,15 +1130,17 @@ public class IssueIndex { } else { componentFilter.filter(termQuery(FIELD_ISSUE_BRANCH_UUID, projectUuid)); } - return client.prepareSearch(TYPE_ISSUE.getMainType()) - .setQuery( + + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + return sourceBuilder + .query( componentFilter .should(NON_RESOLVED_VULNERABILITIES_FILTER) .should(TO_REVIEW_HOTSPOTS_FILTER) .should(IN_REVIEW_HOTSPOTS_FILTER) .should(REVIEWED_HOTSPOTS_FILTER) .minimumShouldMatch(1)) - .setSize(0); + .size(0); } } diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java index 820b2c8033a..4231f0d42be 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java @@ -33,7 +33,6 @@ import java.util.stream.IntStream; import java.util.stream.Stream; import javax.annotation.Nullable; import org.apache.lucene.search.join.ScoreMode; -import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -48,7 +47,8 @@ import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilde import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.sum.Sum; +import org.elasticsearch.search.aggregations.metrics.Sum; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.NestedSortBuilder; import org.sonar.api.measures.Metric; @@ -119,6 +119,8 @@ import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIEL import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_MEASURES_MEASURE_VALUE; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_NAME; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_NCLOC_DISTRIBUTION; +import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_NCLOC_DISTRIBUTION_LANGUAGE; +import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_NCLOC_DISTRIBUTION_NCLOC; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_ORGANIZATION_UUID; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_QUALIFIER; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_QUALITY_GATE_STATUS; @@ -210,20 +212,21 @@ public class ProjectMeasuresIndex { } public SearchIdResult<String> search(ProjectMeasuresQuery query, SearchOptions searchOptions) { - SearchRequestBuilder requestBuilder = client - .prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) - .setFetchSource(false) - .setFrom(searchOptions.getOffset()) - .setSize(searchOptions.getLimit()); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .fetchSource(false) + .from(searchOptions.getOffset()) + .size(searchOptions.getLimit()); AllFilters allFilters = createFilters(query); RequestFiltersComputer filtersComputer = createFiltersComputer(searchOptions, allFilters); - addFacets(requestBuilder, searchOptions, filtersComputer, query); - addSort(query, requestBuilder); - - filtersComputer.getQueryFilters().ifPresent(requestBuilder::setQuery); - filtersComputer.getPostFilters().ifPresent(requestBuilder::setPostFilter); - return new SearchIdResult<>(requestBuilder.get(), id -> id, system2.getDefaultTimeZone()); + addFacets(searchSourceBuilder, searchOptions, filtersComputer, query); + addSort(query, searchSourceBuilder); + + filtersComputer.getQueryFilters().ifPresent(searchSourceBuilder::query); + filtersComputer.getPostFilters().ifPresent(searchSourceBuilder::postFilter); + SearchResponse response = client.search(EsClient.prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) + .source(searchSourceBuilder)); + return new SearchIdResult<>(response, id -> id, system2.getDefaultTimeZone()); } private static RequestFiltersComputer createFiltersComputer(SearchOptions searchOptions, AllFilters allFilters) { @@ -237,39 +240,39 @@ public class ProjectMeasuresIndex { } public ProjectMeasuresStatistics searchTelemetryStatistics() { - SearchRequestBuilder request = client - .prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) - .setFetchSource(false) - .setSize(0); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .fetchSource(false) + .size(0); BoolQueryBuilder esFilter = boolQuery() .filter(termQuery(FIELD_INDEX_TYPE, TYPE_PROJECT_MEASURES.getName())) .filter(termQuery(FIELD_QUALIFIER, Qualifiers.PROJECT)); - request.setQuery(esFilter); - request.addAggregation(AggregationBuilders.terms(FIELD_LANGUAGES) + searchSourceBuilder.query(esFilter); + searchSourceBuilder.aggregation(AggregationBuilders.terms(FIELD_LANGUAGES) .field(FIELD_LANGUAGES) .size(MAX_PAGE_SIZE) .minDocCount(1) .order(BucketOrder.count(false))); - request.addAggregation(AggregationBuilders.nested(FIELD_NCLOC_DISTRIBUTION, FIELD_NCLOC_DISTRIBUTION) + searchSourceBuilder.aggregation(AggregationBuilders.nested(FIELD_NCLOC_DISTRIBUTION, FIELD_NCLOC_DISTRIBUTION) .subAggregation(AggregationBuilders.terms(FIELD_NCLOC_DISTRIBUTION + "_terms") - .field(ProjectMeasuresIndexDefinition.FIELD_NCLOC_DISTRIBUTION_LANGUAGE) + .field(FIELD_NCLOC_DISTRIBUTION_LANGUAGE) .size(MAX_PAGE_SIZE) .minDocCount(1) .order(BucketOrder.count(false)) - .subAggregation(sum(ProjectMeasuresIndexDefinition.FIELD_NCLOC_DISTRIBUTION_NCLOC).field(ProjectMeasuresIndexDefinition.FIELD_NCLOC_DISTRIBUTION_NCLOC)))); + .subAggregation(sum(FIELD_NCLOC_DISTRIBUTION_NCLOC).field(FIELD_NCLOC_DISTRIBUTION_NCLOC)))); - request.addAggregation(AggregationBuilders.nested(NCLOC_KEY, FIELD_MEASURES) + searchSourceBuilder.aggregation(AggregationBuilders.nested(NCLOC_KEY, FIELD_MEASURES) .subAggregation(AggregationBuilders.filter(NCLOC_KEY + "_filter", termQuery(FIELD_MEASURES_MEASURE_KEY, NCLOC_KEY)) .subAggregation(sum(NCLOC_KEY + "_filter_sum").field(FIELD_MEASURES_MEASURE_VALUE)))); ProjectMeasuresStatistics.Builder statistics = ProjectMeasuresStatistics.builder(); - SearchResponse response = request.get(); - statistics.setProjectCount(response.getHits().getTotalHits()); + SearchResponse response = client.search(EsClient.prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) + .source(searchSourceBuilder)); + + statistics.setProjectCount(response.getHits().getTotalHits().value); statistics.setProjectCountByLanguage(termsToMap(response.getAggregations().get(FIELD_LANGUAGES))); - Function<Terms.Bucket, Long> bucketToNcloc = bucket -> Math - .round(((Sum) bucket.getAggregations().get(ProjectMeasuresIndexDefinition.FIELD_NCLOC_DISTRIBUTION_NCLOC)).getValue()); + Function<Terms.Bucket, Long> bucketToNcloc = bucket -> Math.round(((Sum) bucket.getAggregations().get(FIELD_NCLOC_DISTRIBUTION_NCLOC)).getValue()); Map<String, Long> nclocByLanguage = Stream.of((Nested) response.getAggregations().get(FIELD_NCLOC_DISTRIBUTION)) .map(nested -> (Terms) nested.getAggregations().get(nested.getName() + "_terms")) .flatMap(terms -> terms.getBuckets().stream()) @@ -279,25 +282,25 @@ public class ProjectMeasuresIndex { return statistics.build(); } - private static void addSort(ProjectMeasuresQuery query, SearchRequestBuilder requestBuilder) { + private static void addSort(ProjectMeasuresQuery query, SearchSourceBuilder requestBuilder) { String sort = query.getSort(); if (SORT_BY_NAME.equals(sort)) { - requestBuilder.addSort(DefaultIndexSettingsElement.SORTABLE_ANALYZER.subField(FIELD_NAME), query.isAsc() ? ASC : DESC); + requestBuilder.sort(DefaultIndexSettingsElement.SORTABLE_ANALYZER.subField(FIELD_NAME), query.isAsc() ? ASC : DESC); } else if (SORT_BY_LAST_ANALYSIS_DATE.equals(sort)) { - requestBuilder.addSort(FIELD_ANALYSED_AT, query.isAsc() ? ASC : DESC); + requestBuilder.sort(FIELD_ANALYSED_AT, query.isAsc() ? ASC : DESC); } else if (ALERT_STATUS_KEY.equals(sort)) { - requestBuilder.addSort(FIELD_QUALITY_GATE_STATUS, query.isAsc() ? ASC : DESC); - requestBuilder.addSort(DefaultIndexSettingsElement.SORTABLE_ANALYZER.subField(FIELD_NAME), ASC); + requestBuilder.sort(FIELD_QUALITY_GATE_STATUS, query.isAsc() ? ASC : DESC); + requestBuilder.sort(DefaultIndexSettingsElement.SORTABLE_ANALYZER.subField(FIELD_NAME), ASC); } else { addMetricSort(query, requestBuilder, sort); - requestBuilder.addSort(DefaultIndexSettingsElement.SORTABLE_ANALYZER.subField(FIELD_NAME), ASC); + requestBuilder.sort(DefaultIndexSettingsElement.SORTABLE_ANALYZER.subField(FIELD_NAME), ASC); } // last sort is by key in order to be deterministic when same value - requestBuilder.addSort(FIELD_KEY, ASC); + requestBuilder.sort(FIELD_KEY, ASC); } - private static void addMetricSort(ProjectMeasuresQuery query, SearchRequestBuilder requestBuilder, String sort) { - requestBuilder.addSort( + private static void addMetricSort(ProjectMeasuresQuery query, SearchSourceBuilder requestBuilder, String sort) { + requestBuilder.sort( new FieldSortBuilder(FIELD_MEASURES_MEASURE_VALUE) .setNestedSort( new NestedSortBuilder(FIELD_MEASURES) @@ -305,13 +308,13 @@ public class ProjectMeasuresIndex { .order(query.isAsc() ? ASC : DESC)); } - private static void addFacets(SearchRequestBuilder esRequest, SearchOptions options, RequestFiltersComputer filtersComputer, ProjectMeasuresQuery query) { + private static void addFacets(SearchSourceBuilder esRequest, SearchOptions options, RequestFiltersComputer filtersComputer, ProjectMeasuresQuery query) { TopAggregationHelper topAggregationHelper = new TopAggregationHelper(filtersComputer, new SubAggregationHelper()); options.getFacets().stream() .map(FACETS_BY_NAME::get) .filter(Objects::nonNull) .map(facet -> facet.getFacetBuilder().buildFacet(facet, query, topAggregationHelper)) - .forEach(esRequest::addAggregation); + .forEach(esRequest::aggregation); } private static AbstractAggregationBuilder<?> createRangeFacet(String metricKey, double[] thresholds) { @@ -452,14 +455,16 @@ public class ProjectMeasuresIndex { tagFacet.includeExclude(new IncludeExclude(".*" + escapeSpecialRegexChars(textQuery) + ".*", null)); } - SearchRequestBuilder searchQuery = client - .prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) - .setQuery(authorizationTypeSupport.createQueryFilter()) - .setFetchSource(false) - .setSize(0) - .addAggregation(tagFacet); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .query(authorizationTypeSupport.createQueryFilter()) + .fetchSource(false) + .size(0) + .aggregation(tagFacet); + + SearchResponse response = client.search(EsClient.prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) + .source(searchSourceBuilder)); - Terms aggregation = searchQuery.get().getAggregations().get(FIELD_TAGS); + Terms aggregation = response.getAggregations().get(FIELD_TAGS); return aggregation.getBuckets().stream() .map(Bucket::getKeyAsString) diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/es/IndexCreatorTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/es/IndexCreatorTest.java index 8e0fba1e1a3..4d78374bbd7 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/es/IndexCreatorTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/es/IndexCreatorTest.java @@ -24,10 +24,13 @@ import com.google.common.collect.Sets; import java.util.Map; import java.util.function.Consumer; import javax.annotation.CheckForNull; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; -import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import org.junit.Rule; @@ -97,19 +100,20 @@ public class IndexCreatorTest { IndexMainType fakeIndexType = main(Index.simple("fakes"), "fake"); String id = "1"; - es.client().prepareIndex(fakeIndexType).setId(id).setSource(new FakeDoc().getFields()).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); - assertThat(es.client().prepareGet(fakeIndexType, id).get().isExists()).isTrue(); + es.client().index(new IndexRequest(fakeIndexType.getIndex().getName(), fakeIndexType.getType()).id(id).source(new FakeDoc().getFields()) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)); + assertThat(es.client().get(new GetRequest(fakeIndexType.getIndex().getName(), fakeIndexType.getType(), id)).isExists()).isTrue(); // v2 run(new FakeIndexDefinitionV2()); - ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = mappings(); - MappingMetaData mapping = mappings.get("fakes").get("fake"); + ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetadata>> mappings = mappings(); + MappingMetadata mapping = mappings.get("fakes").get("fake"); assertThat(countMappingFields(mapping)).isEqualTo(3); assertThat(field(mapping, "updatedAt").get("type")).isEqualTo("date"); assertThat(field(mapping, "newField").get("type")).isEqualTo("integer"); - assertThat(es.client().prepareGet(fakeIndexType, id).get().isExists()).isFalse(); + assertThat(es.client().get(new GetRequest(fakeIndexType.getIndex().getName(), fakeIndexType.getType(), id)).isExists()).isFalse(); } @Test @@ -207,7 +211,7 @@ public class IndexCreatorTest { private boolean isNotReadOnly(IndexMainType mainType) { String indexName = mainType.getIndex().getName(); - String readOnly = es.client().nativeClient().admin().indices().getSettings(new GetSettingsRequest().indices(indexName)).actionGet() + String readOnly = es.client().getSettings(new GetSettingsRequest().indices(indexName)) .getSetting(indexName, "index.blocks.read_only_allow_delete"); return readOnly == null; } @@ -215,9 +219,7 @@ public class IndexCreatorTest { private void makeReadOnly(IndexMainType mainType) { Settings.Builder builder = Settings.builder(); builder.put("index.blocks.read_only_allow_delete", "true"); - es.client().nativeClient().admin().indices() - .updateSettings(new UpdateSettingsRequest().indices(mainType.getIndex().getName()).settings(builder.build())) - .actionGet(); + es.client().putSettings(new UpdateSettingsRequest().indices(mainType.getIndex().getName()).settings(builder.build())); } private void enableBlueGreenDeployment() { @@ -246,18 +248,18 @@ public class IndexCreatorTest { assertThat(es.countDocuments(FakeIndexDefinition.INDEX_TYPE)).isEqualTo(0); } - private ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings() { - return es.client().nativeClient().admin().indices().prepareGetMappings().get().mappings(); + private ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetadata>> mappings() { + return es.client().getMapping(new GetMappingsRequest()).mappings(); } @CheckForNull @SuppressWarnings("unchecked") - private Map<String, Object> field(MappingMetaData mapping, String field) { + private Map<String, Object> field(MappingMetadata mapping, String field) { Map<String, Object> props = (Map<String, Object>) mapping.getSourceAsMap().get("properties"); return (Map<String, Object>) props.get(field); } - private int countMappingFields(MappingMetaData mapping) { + private int countMappingFields(MappingMetadata mapping) { return ((Map) mapping.getSourceAsMap().get("properties")).size(); } @@ -274,8 +276,8 @@ public class IndexCreatorTest { } private void verifyFakeIndexV1() { - ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = mappings(); - MappingMetaData mapping = mappings.get("fakes").get("fake"); + ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetadata>> mappings = mappings(); + MappingMetadata mapping = mappings.get("fakes").get("fake"); assertThat(mapping.type()).isEqualTo("fake"); assertThat(mapping.getSourceAsMap()).isNotEmpty(); assertThat(countMappingFields(mapping)).isEqualTo(2); diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexDebtTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexDebtTest.java index 835690dfdac..74d67113b28 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexDebtTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexDebtTest.java @@ -23,11 +23,11 @@ import java.util.Map; import java.util.TimeZone; import org.junit.Rule; import org.junit.Test; +import org.sonar.api.impl.utils.TestSystem2; import org.sonar.api.issue.Issue; import org.sonar.api.rule.RuleKey; import org.sonar.api.rule.Severity; import org.sonar.api.utils.System2; -import org.sonar.api.impl.utils.TestSystem2; import org.sonar.db.DbTester; import org.sonar.db.component.ComponentDto; import org.sonar.db.component.ComponentTesting; diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityHotspotsTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityHotspotsTest.java index 23164942c65..054986e98fe 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityHotspotsTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityHotspotsTest.java @@ -28,9 +28,9 @@ import org.elasticsearch.search.SearchHit; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import org.sonar.api.impl.utils.TestSystem2; import org.sonar.api.rule.Severity; import org.sonar.api.utils.System2; -import org.sonar.api.impl.utils.TestSystem2; import org.sonar.db.DbTester; import org.sonar.db.component.ComponentDto; import org.sonar.server.es.EsTester; diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java index b4ba5f4af5d..8563d65c54f 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java @@ -26,6 +26,7 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; +import org.apache.lucene.search.TotalHits; import org.assertj.core.groups.Tuple; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.search.SearchHit; @@ -102,15 +103,15 @@ public class IssueIndexTest { // There are 12 issues in total, with 10 issues per page, the page 2 should only contain 2 elements SearchResponse result = underTest.search(query.build(), new SearchOptions().setPage(2, 10)); assertThat(result.getHits().getHits()).hasSize(2); - assertThat(result.getHits().getTotalHits()).isEqualTo(12); + assertThat(result.getHits().getTotalHits()).isEqualTo(new TotalHits(12, TotalHits.Relation.EQUAL_TO)); result = underTest.search(IssueQuery.builder().build(), new SearchOptions().setOffset(0).setLimit(5)); assertThat(result.getHits().getHits()).hasSize(5); - assertThat(result.getHits().getTotalHits()).isEqualTo(12); + assertThat(result.getHits().getTotalHits()).isEqualTo(new TotalHits(12, TotalHits.Relation.EQUAL_TO)); result = underTest.search(IssueQuery.builder().build(), new SearchOptions().setOffset(2).setLimit(10)); assertThat(result.getHits().getHits()).hasSize(10); - assertThat(result.getHits().getTotalHits()).isEqualTo(12); + assertThat(result.getHits().getTotalHits()).isEqualTo(new TotalHits(12, TotalHits.Relation.EQUAL_TO)); } @Test diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndex.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndex.java index b31117c203e..2011a002881 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndex.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndex.java @@ -21,8 +21,10 @@ package org.sonar.server.permission.index; import java.util.Arrays; import java.util.List; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.sonar.core.util.stream.MoreCollectors; import org.sonar.server.es.EsClient; @@ -40,12 +42,10 @@ public class FooIndex { } public boolean hasAccessToProject(String projectUuid) { - SearchHits hits = esClient.prepareSearch(DESCRIPTOR) - .setTypes(TYPE_AUTHORIZATION.getType()) - .setQuery(QueryBuilders.boolQuery() + SearchHits hits = esClient.search(EsClient.prepareSearch(DESCRIPTOR.getName(), TYPE_AUTHORIZATION.getType()) + .source(new SearchSourceBuilder().query(QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(FooIndexDefinition.FIELD_PROJECT_UUID, projectUuid)) - .filter(authorizationTypeSupport.createQueryFilter())) - .get() + .filter(authorizationTypeSupport.createQueryFilter())))) .getHits(); List<String> names = Arrays.stream(hits.getHits()) .map(h -> h.getSourceAsMap().get(FooIndexDefinition.FIELD_NAME).toString()) diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndexer.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndexer.java index 696d25981c0..c2a2cbfbd47 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndexer.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndexer.java @@ -22,6 +22,7 @@ package org.sonar.server.permission.index; import com.google.common.collect.ImmutableSet; import java.util.Collection; import java.util.Set; +import org.elasticsearch.action.index.IndexRequest; import org.sonar.db.DbSession; import org.sonar.db.es.EsQueueDto; import org.sonar.server.es.BaseDoc; @@ -60,11 +61,11 @@ public class FooIndexer implements ProjectIndexer, NeedAuthorizationIndexer { private void addToIndex(String projectUuid, String name) { FooDoc fooDoc = new FooDoc(projectUuid, name); - esClient.prepareIndex(TYPE_FOO) - .setId(fooDoc.getId()) - .setRouting(fooDoc.getRouting().orElse(null)) - .setSource(fooDoc.getFields()) - .get(); + esClient.index(new IndexRequest(TYPE_FOO.getMainType().getIndex().getName()) + .type(TYPE_FOO.getMainType().getType()) + .id(fooDoc.getId()) + .routing(fooDoc.getRouting().orElse(null)) + .source(fooDoc.getFields())); } private static final class FooDoc extends BaseDoc { diff --git a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/health/EsStatusCheck.java b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/health/EsStatusCheck.java index e2e7c0e7770..b1407678103 100644 --- a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/health/EsStatusCheck.java +++ b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/health/EsStatusCheck.java @@ -19,6 +19,7 @@ */ package org.sonar.server.health; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; @@ -50,7 +51,7 @@ abstract class EsStatusCheck { Health checkEsStatus() { try { - ClusterHealthStatus esStatus = esClient.prepareClusterStats().get().getStatus(); + ClusterHealthStatus esStatus = esClient.clusterHealth(new ClusterHealthRequest()).getStatus(); if (esStatus == null) { return RED_HEALTH_UNAVAILABLE; } diff --git a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/hotspot/ws/SearchAction.java b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/hotspot/ws/SearchAction.java index 41a6b9c46aa..71394bc9a0f 100644 --- a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/hotspot/ws/SearchAction.java +++ b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/hotspot/ws/SearchAction.java @@ -293,7 +293,7 @@ public class SearchAction implements HotspotsWsAction { List<IssueDto> hotspots = toIssueDtos(dbSession, issueKeys); - Paging paging = forPageIndex(wsRequest.getPage()).withPageSize(wsRequest.getIndex()).andTotal((int) result.getHits().getTotalHits()); + Paging paging = forPageIndex(wsRequest.getPage()).withPageSize(wsRequest.getIndex()).andTotal((int) result.getHits().getTotalHits().value); return new SearchResponseData(paging, hotspots); } diff --git a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/issue/ws/SearchAction.java b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/issue/ws/SearchAction.java index ead3f8bdfee..2ea5eb36b18 100644 --- a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/issue/ws/SearchAction.java +++ b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/issue/ws/SearchAction.java @@ -426,7 +426,7 @@ public class SearchAction implements IssuesWsAction { SearchResponseData data = searchResponseLoader.load(preloadedData, collector, additionalFields, facets); // FIXME allow long in Paging - Paging paging = forPageIndex(options.getPage()).withPageSize(options.getLimit()).andTotal((int) result.getHits().getTotalHits()); + Paging paging = forPageIndex(options.getPage()).withPageSize(options.getLimit()).andTotal((int) result.getHits().getTotalHits().value); return searchResponseFormat.formatSearch(additionalFields, data, paging, facets); } diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/health/EsStatusClusterCheckTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/health/EsStatusClusterCheckTest.java index 58668ff407f..53e83883583 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/health/EsStatusClusterCheckTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/health/EsStatusClusterCheckTest.java @@ -31,6 +31,7 @@ import org.sonar.server.es.EsClient; import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.when; @@ -43,7 +44,7 @@ public class EsStatusClusterCheckTest { @Test public void check_ignores_NodeHealth_arg_and_returns_RED_with_cause_if_an_exception_occurs_checking_ES_cluster_status() { Set<NodeHealth> nodeHealths = ImmutableSet.of(newNodeHealth(NodeHealth.Status.GREEN)); - when(esClient.prepareClusterStats()).thenThrow(new RuntimeException("Faking an exception occurring while using the EsClient")); + when(esClient.clusterHealth(any())).thenThrow(new RuntimeException("Faking an exception occurring while using the EsClient")); Health health = new EsStatusClusterCheck(esClient).check(nodeHealths); @@ -54,7 +55,7 @@ public class EsStatusClusterCheckTest { @Test public void check_ignores_NodeHealth_arg_and_returns_GREEN_without_cause_if_ES_cluster_status_is_GREEN() { Set<NodeHealth> nodeHealths = ImmutableSet.of(newNodeHealth(NodeHealth.Status.YELLOW)); - when(esClient.prepareClusterStats().get().getStatus()).thenReturn(ClusterHealthStatus.GREEN); + when(esClient.clusterHealth(any()).getStatus()).thenReturn(ClusterHealthStatus.GREEN); Health health = underTest.check(nodeHealths); diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/health/EsStatusNodeCheckTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/health/EsStatusNodeCheckTest.java index 7786ffaab69..3d7b2641ab7 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/health/EsStatusNodeCheckTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/health/EsStatusNodeCheckTest.java @@ -25,6 +25,7 @@ import org.mockito.Mockito; import org.sonar.server.es.EsClient; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -36,7 +37,7 @@ public class EsStatusNodeCheckTest { @Test public void check_ignores_NodeHealth_arg_and_returns_RED_with_cause_if_an_exception_occurs_checking_ES_cluster_status() { EsClient esClient = mock(EsClient.class); - when(esClient.prepareClusterStats()).thenThrow(new RuntimeException("Faking an exception occurring while using the EsClient")); + when(esClient.clusterHealth(any())).thenThrow(new RuntimeException("Faking an exception occurring while using the EsClient")); Health health = new EsStatusNodeCheck(esClient).check(); @@ -46,7 +47,7 @@ public class EsStatusNodeCheckTest { @Test public void check_returns_GREEN_without_cause_if_ES_cluster_status_is_GREEN() { - when(esClient.prepareClusterStats().get().getStatus()).thenReturn(ClusterHealthStatus.GREEN); + when(esClient.clusterHealth(any()).getStatus()).thenReturn(ClusterHealthStatus.GREEN); Health health = underTest.check(); diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/user/ws/CreateActionTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/user/ws/CreateActionTest.java index 3d5a67462aa..cf16f7afe1a 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/user/ws/CreateActionTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/user/ws/CreateActionTest.java @@ -21,6 +21,8 @@ package org.sonar.server.user.ws; import java.util.HashSet; import java.util.Optional; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -34,6 +36,7 @@ import org.sonar.db.DbTester; import org.sonar.db.user.GroupDto; import org.sonar.db.user.UserDto; import org.sonar.server.authentication.CredentialsLocalAuthentication; +import org.sonar.server.es.EsClient; import org.sonar.server.es.EsTester; import org.sonar.server.exceptions.ForbiddenException; import org.sonar.server.organization.DefaultOrganizationProvider; @@ -109,13 +112,14 @@ public class CreateActionTest { .containsOnly("john", "John", "john@email.com", singletonList("jn"), true); // exists in index - assertThat(es.client().prepareSearch(UserIndexDefinition.TYPE_USER) - .setQuery(boolQuery() - .must(termQuery(FIELD_LOGIN, "john")) - .must(termQuery(FIELD_NAME, "John")) - .must(termQuery(FIELD_EMAIL, "john@email.com")) - .must(termQuery(FIELD_SCM_ACCOUNTS, "jn"))) - .get().getHits().getHits()).hasSize(1); + assertThat(es.client().search(EsClient.prepareSearch(UserIndexDefinition.TYPE_USER) + .source(new SearchSourceBuilder() + .query(boolQuery() + .must(termQuery(FIELD_LOGIN, "john")) + .must(termQuery(FIELD_NAME, "John")) + .must(termQuery(FIELD_EMAIL, "john@email.com")) + .must(termQuery(FIELD_SCM_ACCOUNTS, "jn"))))) + .getHits().getHits()).hasSize(1); // exists in db Optional<UserDto> dbUser = db.users().selectUserByLogin("john"); diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/user/ws/DeactivateActionTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/user/ws/DeactivateActionTest.java index 760f99b1adb..ae1e40c7577 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/user/ws/DeactivateActionTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/user/ws/DeactivateActionTest.java @@ -21,6 +21,8 @@ package org.sonar.server.user.ws; import java.util.Optional; import javax.annotation.Nullable; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -43,6 +45,7 @@ import org.sonar.db.user.GroupDto; import org.sonar.db.user.SessionTokenDto; import org.sonar.db.user.UserDismissedMessageDto; import org.sonar.db.user.UserDto; +import org.sonar.server.es.EsClient; import org.sonar.server.es.EsTester; import org.sonar.server.exceptions.BadRequestException; import org.sonar.server.exceptions.ForbiddenException; @@ -100,11 +103,12 @@ public class DeactivateActionTest { deactivate(user.getLogin()); verifyThatUserIsDeactivated(user.getLogin()); - assertThat(es.client().prepareSearch(UserIndexDefinition.TYPE_USER) - .setQuery(boolQuery() - .must(termQuery(FIELD_UUID, user.getUuid())) - .must(termQuery(FIELD_ACTIVE, "false"))) - .get().getHits().getHits()).hasSize(1); + assertThat(es.client().search(EsClient.prepareSearch(UserIndexDefinition.TYPE_USER) + .source(new SearchSourceBuilder() + .query(boolQuery() + .must(termQuery(FIELD_UUID, user.getUuid())) + .must(termQuery(FIELD_ACTIVE, "false"))))) + .getHits().getHits()).hasSize(1); } @Test diff --git a/sonar-application/build.gradle b/sonar-application/build.gradle index e9a4d6d8952..4b433567978 100644 --- a/sonar-application/build.gradle +++ b/sonar-application/build.gradle @@ -30,7 +30,7 @@ dependencies { // please keep list ordered compile 'org.slf4j:slf4j-api' - compile 'org.elasticsearch.client:transport' + compile 'org.elasticsearch.client:elasticsearch-rest-high-level-client' compile project(':server:sonar-ce') compile project(':server:sonar-main') compile project(':server:sonar-process') diff --git a/sonar-application/src/main/assembly/conf/sonar.properties b/sonar-application/src/main/assembly/conf/sonar.properties index 6ce66482775..6ca7452a06e 100644 --- a/sonar-application/src/main/assembly/conf/sonar.properties +++ b/sonar-application/src/main/assembly/conf/sonar.properties @@ -271,7 +271,7 @@ # Same as previous property, but allows to not repeat all other settings like -Xmx #sonar.search.javaAdditionalOpts= -# Elasticsearch port. Default is 9001. Use 0 to get a free port. +# Elasticsearch port for incoming HTTP connections. Default is 9001. Use 0 to get a free port. # As a security precaution, should be blocked by a firewall and not exposed to the Internet. #sonar.search.port=9001 @@ -413,11 +413,3 @@ # We don't collect source code or IP addresses. And we don't share the data with anyone else. # To see an example of the data shared: login as a global administrator, call the WS api/system/info and check the Statistics field. #sonar.telemetry.enable=true - - -#-------------------------------------------------------------------------------------------------- -# DEVELOPMENT - only for developers -# The following properties MUST NOT be used in production environments. - -# Elasticsearch HTTP connector -#sonar.search.httpPort=-1 diff --git a/sonar-application/src/main/java/org/sonar/application/App.java b/sonar-application/src/main/java/org/sonar/application/App.java index 5ab3ae12142..d7f7341aaf0 100644 --- a/sonar-application/src/main/java/org/sonar/application/App.java +++ b/sonar-application/src/main/java/org/sonar/application/App.java @@ -42,7 +42,6 @@ public class App { private final JavaVersion javaVersion; private StopRequestWatcher stopRequestWatcher = null; private StopRequestWatcher hardStopRequestWatcher = null; - public App(JavaVersion javaVersion) { this.javaVersion = javaVersion; } |