SONAR-8798 replace any usage of SearchType.SCAN by a scroll query

diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java b/server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java
index 7fe8e4205bf9e912e36bb8e1f1766d0120c0efe2..0ed12e5a20ec110c61a4be57aa01cdc98a0520e7 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java
@@ -33,11 +33,14 @@ import java.util.function.Function;
 import java.util.regex.Pattern;
 import javax.annotation.CheckForNull;
 import javax.annotation.Nullable;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchScrollRequestBuilder;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+import org.elasticsearch.search.sort.SortOrder;
 import org.joda.time.format.ISODateTimeFormat;
 import org.sonar.core.util.stream.MoreCollectors;
 
@@ -95,8 +98,12 @@ public class EsUtils {
     return null;
   }
 
-  public static <D extends BaseDoc> Iterator<D> scroll(EsClient esClient, String scrollId, Function<Map<String, Object>, D> docConverter) {
-    return new DocScrollIterator<>(esClient, scrollId, docConverter);
+  /**
+   * Optimize scolling, by specifying document sorting.
+   * See https://www.elastic.co/guide/en/elasticsearch/reference/2.4/search-request-scroll.html#search-request-scroll
+   */
+  public static void optimizeScrollRequest(SearchRequestBuilder esSearch) {
+    esSearch.addSort("_doc", SortOrder.ASC);
   }
 
   /**
@@ -116,10 +123,11 @@ public class EsUtils {
 
     private final Queue<SearchHit> hits = new ArrayDeque<>();
 
-    private DocScrollIterator(EsClient esClient, String scrollId, Function<Map<String, Object>, D> docConverter) {
+    private DocScrollIterator(EsClient esClient, SearchResponse scrollResponse, Function<Map<String, Object>, D> docConverter) {
       this.esClient = esClient;
-      this.scrollId = scrollId;
+      this.scrollId = scrollResponse.getScrollId();
       this.docConverter = docConverter;
+      Collections.addAll(hits, scrollResponse.getHits().getHits());
     }
 
     @Override
@@ -146,8 +154,8 @@ public class EsUtils {
     }
   }
 
-  public static <ID> Iterator<ID> scrollIds(EsClient esClient, String scrollId, Function<String, ID> idConverter) {
-    return new IdScrollIterator<>(esClient, scrollId, idConverter);
+  public static <ID> Iterator<ID> scrollIds(EsClient esClient, SearchResponse scrollResponse, Function<String, ID> idConverter) {
+    return new IdScrollIterator<>(esClient, scrollResponse, idConverter);
   }
 
   private static class IdScrollIterator<ID> implements Iterator<ID> {
@@ -158,10 +166,11 @@ public class EsUtils {
 
     private final Queue<SearchHit> hits = new ArrayDeque<>();
 
-    private IdScrollIterator(EsClient esClient, String scrollId, Function<String, ID> idConverter) {
+    private IdScrollIterator(EsClient esClient, SearchResponse scrollResponse, Function<String, ID> idConverter) {
       this.esClient = esClient;
-      this.scrollId = scrollId;
+      this.scrollId = scrollResponse.getScrollId();
       this.idConverter = idConverter;
+      Collections.addAll(hits, scrollResponse.getHits().getHits());
     }
 
     @Override

diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java
index fc2fd918bb48d07142b48ff2be6cf5fb970b91f2..e1774ec72e1deb05ef8ec750fc0382670d76ad85 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java
@@ -32,7 +32,6 @@ import javax.annotation.Nullable;
 import org.apache.commons.lang.StringUtils;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.HasParentQueryBuilder;
@@ -77,6 +76,7 @@ import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_WORDS_ANALY
 import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
 import static org.sonar.server.es.EsUtils.SCROLL_TIME_IN_MINUTES;
 import static org.sonar.server.es.EsUtils.escapeSpecialRegexChars;
+import static org.sonar.server.es.EsUtils.optimizeScrollRequest;
 import static org.sonar.server.es.EsUtils.scrollIds;
 import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_INHERITANCE;
 import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_PROFILE_UUID;
@@ -161,9 +161,9 @@ public class RuleIndex {
   public Iterator<RuleKey> searchAll(RuleQuery query) {
     SearchRequestBuilder esSearch = client
       .prepareSearch(INDEX_TYPE_RULE)
-      .setSearchType(SearchType.SCAN)
       .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES));
 
+    optimizeScrollRequest(esSearch);
     QueryBuilder qb = buildQuery(query);
     Map<String, QueryBuilder> filters = buildFilters(query);
 
@@ -174,7 +174,7 @@ public class RuleIndex {
 
     esSearch.setQuery(boolQuery().must(qb).filter(fb));
     SearchResponse response = esSearch.get();
-    return scrollIds(client, response.getScrollId(), RuleKey::parse);
+    return scrollIds(client, response, RuleKey::parse);
   }
 
   /* Build main query (search based) */

diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java b/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java
index c0a1329b5149b00818de8adc805ce33257b2997a..567902214e2221e2ea9346c923f77d56f2b54f2a 100644 (file)
--- a/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java
@@ -40,7 +40,6 @@ import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.unit.TimeValue;
@@ -157,8 +156,8 @@ public class EsTester extends ExternalResource {
    */
   public List<SearchHit> getDocuments(IndexType indexType) {
     SearchRequestBuilder req = client.nativeClient().prepareSearch(indexType.getIndex()).setTypes(indexType.getType()).setQuery(QueryBuilders.matchAllQuery());
-    req.setSearchType(SearchType.SCAN)
-      .setScroll(new TimeValue(60000))
+    EsUtils.optimizeScrollRequest(req);
+    req.setScroll(new TimeValue(60000))
       .setSize(100);
 
     SearchResponse response = req.get();

diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilderTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilderTest.java
index 6235708e1e5fbab02c3ecc996dbac326d41b26fd..ba98c0981cc092c34357fceeceb9f364c57716c3 100644 (file)
--- a/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilderTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilderTest.java
@@ -20,7 +20,6 @@
 package org.sonar.server.es.request;
 
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.common.unit.TimeValue;
 import org.junit.Rule;
 import org.junit.Test;
@@ -45,7 +44,6 @@ public class ProxySearchScrollRequestBuilderTest {
     logTester.setLevel(LoggerLevel.TRACE);
 
     SearchResponse response = esTester.client().prepareSearch(FakeIndexDefinition.INDEX)
-      .setSearchType(SearchType.SCAN)
       .setScroll(TimeValue.timeValueMinutes(1))
       .get();
     logTester.clear();
@@ -58,7 +56,6 @@ public class ProxySearchScrollRequestBuilderTest {
     logTester.setLevel(LoggerLevel.DEBUG);
 
     SearchResponse response = esTester.client().prepareSearch(FakeIndexDefinition.INDEX)
-      .setSearchType(SearchType.SCAN)
       .setScroll(TimeValue.timeValueMinutes(1))
       .get();
     logTester.clear();