From: Daniel Schwarz Date: Wed, 28 Jun 2017 12:23:32 +0000 (+0200) Subject: SONAR-8798 replace any usage of SearchType.SCAN by a scroll query X-Git-Tag: 6.6-RC1~720 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=07c136c8fc5926bfe26f9fc05b6b8a82e5e3d2a3;p=sonarqube.git SONAR-8798 replace any usage of SearchType.SCAN by a scroll query --- diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java b/server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java index 7fe8e4205bf..0ed12e5a20e 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java @@ -33,11 +33,14 @@ import java.util.function.Function; import java.util.regex.Pattern; import javax.annotation.CheckForNull; import javax.annotation.Nullable; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequestBuilder; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.sort.SortOrder; import org.joda.time.format.ISODateTimeFormat; import org.sonar.core.util.stream.MoreCollectors; @@ -95,8 +98,12 @@ public class EsUtils { return null; } - public static Iterator scroll(EsClient esClient, String scrollId, Function, D> docConverter) { - return new DocScrollIterator<>(esClient, scrollId, docConverter); + /** + * Optimize scolling, by specifying document sorting. + * See https://www.elastic.co/guide/en/elasticsearch/reference/2.4/search-request-scroll.html#search-request-scroll + */ + public static void optimizeScrollRequest(SearchRequestBuilder esSearch) { + esSearch.addSort("_doc", SortOrder.ASC); } /** @@ -116,10 +123,11 @@ public class EsUtils { private final Queue hits = new ArrayDeque<>(); - private DocScrollIterator(EsClient esClient, String scrollId, Function, D> docConverter) { + private DocScrollIterator(EsClient esClient, SearchResponse scrollResponse, Function, D> docConverter) { this.esClient = esClient; - this.scrollId = scrollId; + this.scrollId = scrollResponse.getScrollId(); this.docConverter = docConverter; + Collections.addAll(hits, scrollResponse.getHits().getHits()); } @Override @@ -146,8 +154,8 @@ public class EsUtils { } } - public static Iterator scrollIds(EsClient esClient, String scrollId, Function idConverter) { - return new IdScrollIterator<>(esClient, scrollId, idConverter); + public static Iterator scrollIds(EsClient esClient, SearchResponse scrollResponse, Function idConverter) { + return new IdScrollIterator<>(esClient, scrollResponse, idConverter); } private static class IdScrollIterator implements Iterator { @@ -158,10 +166,11 @@ public class EsUtils { private final Queue hits = new ArrayDeque<>(); - private IdScrollIterator(EsClient esClient, String scrollId, Function idConverter) { + private IdScrollIterator(EsClient esClient, SearchResponse scrollResponse, Function idConverter) { this.esClient = esClient; - this.scrollId = scrollId; + this.scrollId = scrollResponse.getScrollId(); this.idConverter = idConverter; + Collections.addAll(hits, scrollResponse.getHits().getHits()); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java index fc2fd918bb4..e1774ec72e1 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java @@ -32,7 +32,6 @@ import javax.annotation.Nullable; import org.apache.commons.lang.StringUtils; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.HasParentQueryBuilder; @@ -77,6 +76,7 @@ import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_WORDS_ANALY import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER; import static org.sonar.server.es.EsUtils.SCROLL_TIME_IN_MINUTES; import static org.sonar.server.es.EsUtils.escapeSpecialRegexChars; +import static org.sonar.server.es.EsUtils.optimizeScrollRequest; import static org.sonar.server.es.EsUtils.scrollIds; import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_INHERITANCE; import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_PROFILE_UUID; @@ -161,9 +161,9 @@ public class RuleIndex { public Iterator searchAll(RuleQuery query) { SearchRequestBuilder esSearch = client .prepareSearch(INDEX_TYPE_RULE) - .setSearchType(SearchType.SCAN) .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)); + optimizeScrollRequest(esSearch); QueryBuilder qb = buildQuery(query); Map filters = buildFilters(query); @@ -174,7 +174,7 @@ public class RuleIndex { esSearch.setQuery(boolQuery().must(qb).filter(fb)); SearchResponse response = esSearch.get(); - return scrollIds(client, response.getScrollId(), RuleKey::parse); + return scrollIds(client, response, RuleKey::parse); } /* Build main query (search based) */ diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java b/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java index c0a1329b514..567902214e2 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java @@ -40,7 +40,6 @@ import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchType; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.unit.TimeValue; @@ -157,8 +156,8 @@ public class EsTester extends ExternalResource { */ public List getDocuments(IndexType indexType) { SearchRequestBuilder req = client.nativeClient().prepareSearch(indexType.getIndex()).setTypes(indexType.getType()).setQuery(QueryBuilders.matchAllQuery()); - req.setSearchType(SearchType.SCAN) - .setScroll(new TimeValue(60000)) + EsUtils.optimizeScrollRequest(req); + req.setScroll(new TimeValue(60000)) .setSize(100); SearchResponse response = req.get(); diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilderTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilderTest.java index 6235708e1e5..ba98c0981cc 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilderTest.java @@ -20,7 +20,6 @@ package org.sonar.server.es.request; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.unit.TimeValue; import org.junit.Rule; import org.junit.Test; @@ -45,7 +44,6 @@ public class ProxySearchScrollRequestBuilderTest { logTester.setLevel(LoggerLevel.TRACE); SearchResponse response = esTester.client().prepareSearch(FakeIndexDefinition.INDEX) - .setSearchType(SearchType.SCAN) .setScroll(TimeValue.timeValueMinutes(1)) .get(); logTester.clear(); @@ -58,7 +56,6 @@ public class ProxySearchScrollRequestBuilderTest { logTester.setLevel(LoggerLevel.DEBUG); SearchResponse response = esTester.client().prepareSearch(FakeIndexDefinition.INDEX) - .setSearchType(SearchType.SCAN) .setScroll(TimeValue.timeValueMinutes(1)) .get(); logTester.clear();