From a06e719ca8456869902d57a15a0c430fa16be8ca Mon Sep 17 00:00:00 2001 From: Stephane Gamard Date: Tue, 30 Sep 2014 17:22:35 +0200 Subject: [PATCH] SONAR-5548 - Moved ES analyzer definition to indices rather than to node configuration (permits node transport) --- .../java/org/sonar/search/SearchServer.java | 61 +------------ .../server/activity/index/ActivityIndex.java | 25 +++--- .../issue/index/IssueAuthorizationIndex.java | 11 --- .../sonar/server/issue/index/IssueIndex.java | 31 +++---- .../qualityprofile/index/ActiveRuleIndex.java | 21 ++--- .../sonar/server/rule/index/RuleIndex.java | 52 ++++++----- .../org/sonar/server/search/BaseIndex.java | 89 +++++++++++++++++-- .../sonar/server/search/BaseIndexTest.java | 9 -- 8 files changed, 151 insertions(+), 148 deletions(-) diff --git a/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java b/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java index 2f556aeb1b9..860d5c33423 100644 --- a/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java +++ b/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java @@ -134,7 +134,7 @@ public class SearchServer implements Monitored { // Set cluster coordinates esSettings.put("cluster.name", clusterName); esSettings.put("node.rack_id", props.value(SONAR_NODE_NAME, "unknown")); - esSettings.put("cluster.routing.allocation.awareness.attributes", "rack_id"); + // esSettings.put("cluster.routing.allocation.awareness.attributes", "rack_id"); if (props.contains(SONAR_NODE_NAME)) { esSettings.put("node.name", props.value(SONAR_NODE_NAME)); } else { @@ -146,9 +146,6 @@ public class SearchServer implements Monitored { } } - // Make sure the index settings are up to date. - initAnalysis(esSettings); - // And building our ES Node node = NodeBuilder.nodeBuilder() .settings(esSettings) @@ -181,62 +178,6 @@ public class SearchServer implements Monitored { } } - private void initAnalysis(ImmutableSettings.Builder esSettings) { - esSettings - - // Disallow dynamic mapping (too expensive) - .put("index.mapper.dynamic", false) - - // Sortable text analyzer - .put("index.analysis.analyzer.sortable.type", "custom") - .put("index.analysis.analyzer.sortable.tokenizer", "keyword") - .putArray("index.analysis.analyzer.sortable.filter", "trim", "lowercase", "truncate") - - // Edge NGram index-analyzer - .put("index.analysis.analyzer.index_grams.type", "custom") - .put("index.analysis.analyzer.index_grams.tokenizer", "whitespace") - .putArray("index.analysis.analyzer.index_grams.filter", "trim", "lowercase", "gram_filter") - - // Edge NGram search-analyzer - .put("index.analysis.analyzer.search_grams.type", "custom") - .put("index.analysis.analyzer.search_grams.tokenizer", "whitespace") - .putArray("index.analysis.analyzer.search_grams.filter", "trim", "lowercase") - - // Word index-analyzer - .put("index.analysis.analyzer.index_words.type", "custom") - .put("index.analysis.analyzer.index_words.tokenizer", "standard") - .putArray("index.analysis.analyzer.index_words.filter", - "standard", "word_filter", "lowercase", "stop", "asciifolding", "porter_stem") - - // Word search-analyzer - .put("index.analysis.analyzer.search_words.type", "custom") - .put("index.analysis.analyzer.search_words.tokenizer", "standard") - .putArray("index.analysis.analyzer.search_words.filter", - "standard", "lowercase", "stop", "asciifolding", "porter_stem") - - // Edge NGram filter - .put("index.analysis.filter.gram_filter.type", "edgeNGram") - .put("index.analysis.filter.gram_filter.min_gram", 2) - .put("index.analysis.filter.gram_filter.max_gram", 15) - .putArray("index.analysis.filter.gram_filter.token_chars", "letter", "digit", "punctuation", "symbol") - - // Word filter - .put("index.analysis.filter.word_filter.type", "word_delimiter") - .put("index.analysis.filter.word_filter.generate_word_parts", true) - .put("index.analysis.filter.word_filter.catenate_words", true) - .put("index.analysis.filter.word_filter.catenate_numbers", true) - .put("index.analysis.filter.word_filter.catenate_all", true) - .put("index.analysis.filter.word_filter.split_on_case_change", true) - .put("index.analysis.filter.word_filter.preserve_original", true) - .put("index.analysis.filter.word_filter.split_on_numerics", true) - .put("index.analysis.filter.word_filter.stem_english_possessive", true) - - // Path Analyzer - .put("index.analysis.analyzer.path_analyzer.type", "custom") - .put("index.analysis.analyzer.path_analyzer.tokenizer", "path_hierarchy"); - - } - private File esHomeDir() { return props.nonNullValueAsFile(SONAR_PATH_HOME); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndex.java b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndex.java index caf237b441f..45950738e4c 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndex.java @@ -23,17 +23,23 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.*; +import org.elasticsearch.index.query.AndFilterBuilder; +import org.elasticsearch.index.query.FilterBuilder; +import org.elasticsearch.index.query.FilterBuilders; +import org.elasticsearch.index.query.OrFilterBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.sort.SortOrder; import org.sonar.core.activity.Activity; import org.sonar.core.activity.db.ActivityDto; -import org.sonar.server.search.*; +import org.sonar.server.search.BaseIndex; +import org.sonar.server.search.IndexDefinition; +import org.sonar.server.search.IndexField; +import org.sonar.server.search.QueryContext; +import org.sonar.server.search.Result; +import org.sonar.server.search.SearchClient; import javax.annotation.Nullable; - -import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -59,12 +65,9 @@ public class ActivityIndex extends BaseIndex { } @Override - protected Settings getIndexSettings() throws IOException { - return ImmutableSettings.builder() - .put("index.number_of_replicas", 0) - .put("index.number_of_shards", 1) - .put("analysis.analyzer.default.type", "keyword") - .build(); + protected ImmutableSettings.Builder addCustomIndexSettings(ImmutableSettings.Builder settings) { + return settings + .put("analysis.analyzer.default.type", "keyword"); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueAuthorizationIndex.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueAuthorizationIndex.java index c746fb2523c..589f437fb38 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueAuthorizationIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueAuthorizationIndex.java @@ -21,15 +21,12 @@ package org.sonar.server.issue.index; import com.google.common.base.Preconditions; -import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.common.settings.Settings; import org.sonar.core.issue.db.IssueAuthorizationDto; import org.sonar.server.search.BaseIndex; import org.sonar.server.search.IndexDefinition; import org.sonar.server.search.IndexField; import org.sonar.server.search.SearchClient; -import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -44,14 +41,6 @@ public class IssueAuthorizationIndex extends BaseIndex mapping = new HashMap(); diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java index 72daf82ed87..e8fc75ffd39 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java @@ -25,10 +25,12 @@ import org.apache.commons.lang.BooleanUtils; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.*; +import org.elasticsearch.index.query.BoolFilterBuilder; +import org.elasticsearch.index.query.FilterBuilders; +import org.elasticsearch.index.query.OrFilterBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortBuilders; @@ -37,12 +39,19 @@ import org.sonar.api.issue.Issue; import org.sonar.api.issue.IssueQuery; import org.sonar.api.web.UserRole; import org.sonar.core.issue.db.IssueDto; -import org.sonar.server.search.*; +import org.sonar.server.search.BaseIndex; +import org.sonar.server.search.IndexDefinition; +import org.sonar.server.search.IndexField; +import org.sonar.server.search.QueryContext; +import org.sonar.server.search.Result; +import org.sonar.server.search.SearchClient; import javax.annotation.Nullable; - -import java.io.IOException; -import java.util.*; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; import static com.google.common.collect.Lists.newArrayList; @@ -57,14 +66,6 @@ public class IssueIndex extends BaseIndex { return keyString; } - @Override - protected Settings getIndexSettings() throws IOException { - return ImmutableSettings.builder() - .put("index.number_of_replicas", 0) - .put("index.number_of_shards", 1) - .build(); - } - @Override protected Map mapProperties() { Map mapping = new HashMap(); diff --git a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java index e96cafcecba..80576ddfb78 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java @@ -23,8 +23,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Multimap; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.FilterBuilders; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; @@ -37,9 +35,12 @@ import org.sonar.core.qualityprofile.db.ActiveRuleDto; import org.sonar.core.qualityprofile.db.ActiveRuleKey; import org.sonar.server.qualityprofile.ActiveRule; import org.sonar.server.rule.index.RuleNormalizer; -import org.sonar.server.search.*; +import org.sonar.server.search.BaseIndex; +import org.sonar.server.search.FacetValue; +import org.sonar.server.search.IndexDefinition; +import org.sonar.server.search.IndexField; +import org.sonar.server.search.SearchClient; -import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -56,14 +57,6 @@ public class ActiveRuleIndex extends BaseIndex mapping = new HashMap(); @@ -120,7 +113,7 @@ public class ActiveRuleIndex extends BaseIndex activeRules = new ArrayList(); diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java index 9811650de10..3523f1d46ea 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java @@ -24,10 +24,15 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequestBuilder; import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.*; +import org.elasticsearch.index.query.BoolFilterBuilder; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.FilterBuilder; +import org.elasticsearch.index.query.FilterBuilders; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.SimpleQueryStringBuilder; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.terms.Terms; @@ -40,12 +45,21 @@ import org.sonar.api.server.debt.DebtCharacteristic; import org.sonar.core.rule.RuleDto; import org.sonar.server.qualityprofile.index.ActiveRuleNormalizer; import org.sonar.server.rule.Rule; -import org.sonar.server.search.*; +import org.sonar.server.search.BaseIndex; +import org.sonar.server.search.IndexDefinition; +import org.sonar.server.search.IndexField; +import org.sonar.server.search.QueryContext; +import org.sonar.server.search.Result; +import org.sonar.server.search.SearchClient; import javax.annotation.CheckForNull; - -import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import static com.google.common.collect.Lists.newArrayList; @@ -59,14 +73,6 @@ public class RuleIndex extends BaseIndex { return key.toString(); } - @Override - protected Settings getIndexSettings() throws IOException { - return ImmutableSettings.builder() - .put("index.number_of_replicas", 0) - .put("index.number_of_shards", 1) - .build(); - } - @Override protected Map mapKey() { Map mapping = new HashMap(); @@ -165,10 +171,10 @@ public class RuleIndex extends BaseIndex { // Human readable type of querying qb.should(QueryBuilders.simpleQueryString(query.getQueryText()) - .field(RuleNormalizer.RuleField.NAME.field() + "." + IndexField.SEARCH_WORDS_SUFFIX, 20f) - .field(RuleNormalizer.RuleField.HTML_DESCRIPTION.field() + "." + IndexField.SEARCH_WORDS_SUFFIX, 3f) - .defaultOperator(SimpleQueryStringBuilder.Operator.AND) - ).boost(20f); + .field(RuleNormalizer.RuleField.NAME.field() + "." + IndexField.SEARCH_WORDS_SUFFIX, 20f) + .field(RuleNormalizer.RuleField.HTML_DESCRIPTION.field() + "." + IndexField.SEARCH_WORDS_SUFFIX, 3f) + .defaultOperator(SimpleQueryStringBuilder.Operator.AND) + ).boost(20f); // Match and partial Match queries qb.should(this.termQuery(RuleNormalizer.RuleField.KEY, queryString, 15f)); @@ -214,7 +220,7 @@ public class RuleIndex extends BaseIndex { FilterBuilders.orFilter( FilterBuilders.termsFilter(RuleNormalizer.RuleField.SUB_CHARACTERISTIC.field(), debtCharacteristics), FilterBuilders.termsFilter(RuleNormalizer.RuleField.CHARACTERISTIC.field(), debtCharacteristics)) - ), + ), // Match only when NOT NONE (not overridden) FilterBuilders.andFilter( @@ -224,8 +230,8 @@ public class RuleIndex extends BaseIndex { FilterBuilders.orFilter( FilterBuilders.termsFilter(RuleNormalizer.RuleField.DEFAULT_SUB_CHARACTERISTIC.field(), debtCharacteristics), FilterBuilders.termsFilter(RuleNormalizer.RuleField.DEFAULT_CHARACTERISTIC.field(), debtCharacteristics))) - ) - ); + ) + ); } // Debt char exist filter @@ -416,7 +422,7 @@ public class RuleIndex extends BaseIndex { for (SearchHit hit : scrollResp.getHits()) { rules.add(toDoc(hit.getSource())); } - //Break condition: No hits are returned + // Break condition: No hits are returned if (scrollResp.getHits().getHits().length == 0) { break; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java b/server/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java index 38dca532a4b..8741898e106 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java @@ -26,10 +26,16 @@ import com.google.common.collect.Multimap; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.elasticsearch.action.count.CountRequestBuilder; import org.elasticsearch.action.count.CountResponse; -import org.elasticsearch.action.get.*; +import org.elasticsearch.action.get.GetRequestBuilder; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.get.MultiGetItemResponse; +import org.elasticsearch.action.get.MultiGetRequest; +import org.elasticsearch.action.get.MultiGetRequestBuilder; +import org.elasticsearch.action.get.MultiGetResponse; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequestBuilder; +import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.BoolFilterBuilder; @@ -53,10 +59,17 @@ import org.sonar.server.exceptions.NotFoundException; import javax.annotation.CheckForNull; import javax.annotation.Nullable; - -import java.io.IOException; import java.io.Serializable; -import java.util.*; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Queue; public abstract class BaseIndex, KEY extends Serializable> implements Index { @@ -218,7 +231,14 @@ public abstract class BaseIndex, KEY extends Serial protected abstract String getKeyValue(KEY key); - protected abstract Settings getIndexSettings() throws IOException; + private final Settings getIndexSettings() { + return this.addCustomIndexSettings + (this.getBaseIndexSettings()).build(); + } + + protected ImmutableSettings.Builder addCustomIndexSettings(ImmutableSettings.Builder baseIndexSettings) { + return baseIndexSettings; + } protected abstract Map mapProperties(); @@ -497,4 +517,63 @@ public abstract class BaseIndex, KEY extends Serial } return stats; } + + private ImmutableSettings.Builder getBaseIndexSettings() { + return ImmutableSettings.builder() + + .put("index.number_of_replicas", 0) + .put("index.number_of_shards", 1) + + // Disallow dynamic mapping (too expensive) + .put("index.mapper.dynamic", false) + + // Sortable text analyzer + .put("index.analysis.analyzer.sortable.type", "custom") + .put("index.analysis.analyzer.sortable.tokenizer", "keyword") + .putArray("index.analysis.analyzer.sortable.filter", "trim", "lowercase", "truncate") + + // Edge NGram index-analyzer + .put("index.analysis.analyzer.index_grams.type", "custom") + .put("index.analysis.analyzer.index_grams.tokenizer", "whitespace") + .putArray("index.analysis.analyzer.index_grams.filter", "trim", "lowercase", "gram_filter") + + // Edge NGram search-analyzer + .put("index.analysis.analyzer.search_grams.type", "custom") + .put("index.analysis.analyzer.search_grams.tokenizer", "whitespace") + .putArray("index.analysis.analyzer.search_grams.filter", "trim", "lowercase") + + // Word index-analyzer + .put("index.analysis.analyzer.index_words.type", "custom") + .put("index.analysis.analyzer.index_words.tokenizer", "standard") + .putArray("index.analysis.analyzer.index_words.filter", + "standard", "word_filter", "lowercase", "stop", "asciifolding", "porter_stem") + + // Word search-analyzer + .put("index.analysis.analyzer.search_words.type", "custom") + .put("index.analysis.analyzer.search_words.tokenizer", "standard") + .putArray("index.analysis.analyzer.search_words.filter", + "standard", "lowercase", "stop", "asciifolding", "porter_stem") + + // Edge NGram filter + .put("index.analysis.filter.gram_filter.type", "edgeNGram") + .put("index.analysis.filter.gram_filter.min_gram", 2) + .put("index.analysis.filter.gram_filter.max_gram", 15) + .putArray("index.analysis.filter.gram_filter.token_chars", "letter", "digit", "punctuation", "symbol") + + // Word filter + .put("index.analysis.filter.word_filter.type", "word_delimiter") + .put("index.analysis.filter.word_filter.generate_word_parts", true) + .put("index.analysis.filter.word_filter.catenate_words", true) + .put("index.analysis.filter.word_filter.catenate_numbers", true) + .put("index.analysis.filter.word_filter.catenate_all", true) + .put("index.analysis.filter.word_filter.split_on_case_change", true) + .put("index.analysis.filter.word_filter.preserve_original", true) + .put("index.analysis.filter.word_filter.split_on_numerics", true) + .put("index.analysis.filter.word_filter.stem_english_possessive", true) + + // Path Analyzer + .put("index.analysis.analyzer.path_analyzer.type", "custom") + .put("index.analysis.analyzer.path_analyzer.tokenizer", "path_hierarchy"); + + } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/BaseIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/BaseIndexTest.java index 2520f83c2ab..7cb1af52a8c 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/BaseIndexTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/BaseIndexTest.java @@ -19,9 +19,7 @@ */ package org.sonar.server.search; - import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; -import org.elasticsearch.common.settings.ImmutableSettings; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -52,7 +50,6 @@ public class BaseIndexTest { private static String clusterName; private static Integer clusterPort; - @BeforeClass public static void setupSearchEngine() { clusterName = "cluster-mem-" + System.currentTimeMillis(); @@ -100,7 +97,6 @@ public class BaseIndexTest { assertThat(indexExistsResponse.isExists()).isTrue(); } - private BaseIndex getIndex(final SearchClient searchClient) { BaseIndex index = new BaseIndex( IndexDefinition.TEST, @@ -110,11 +106,6 @@ public class BaseIndexTest { return null; } - @Override - protected org.elasticsearch.common.settings.Settings getIndexSettings() throws IOException { - return ImmutableSettings.builder().build(); - } - @Override protected Map mapProperties() { return Collections.emptyMap(); -- 2.39.5