SONAR-8067 do not override default Elasticsearch doc_values

Default ES behavior should be applied (doc_values is used
on any possible field in ES 2.x)

diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java
index 9ea547d499f78780bf077b6f2b470311cc3447ef..6c0cac7256522de3cd956072f3d9f5458ed35434 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java
@@ -185,9 +185,7 @@ public class NewIndex {
   public static class StringFieldBuilder {
     private final NewIndexType indexType;
     private final String fieldName;
-    private boolean docValues = false;
     private boolean disableSearch = false;
-    private boolean hasAnalyzedField = false;
     private boolean disableNorms = false;
     private SortedMap<String, Object> subFields = Maps.newTreeMap();
 
@@ -196,27 +194,12 @@ public class NewIndex {
       this.fieldName = fieldName;
     }
 
-    /**
-     * It is recommended to enable doc_values when the non-analyzed field is involved
-     * into aggregations/sorting and only a small fraction of the values is used (high
-     * number of different values)
-     * <ul>
-     *   <li>https://www.elastic.co/blog/found-sizing-elasticsearch</li>
-     *   <li>https://www.elastic.co/guide/en/elasticsearch/guide/current/doc-values.html</li>
-     * </ul>
-     */
-    public StringFieldBuilder docValues() {
-      this.docValues = true;
-      return this;
-    }
-
     /**
      * Add a sub-field. A {@code SortedMap} is required for consistency of the index settings hash.
      * @see IndexDefinitionHash
      */
     public StringFieldBuilder addSubField(String fieldName, SortedMap<String, String> fieldDefinition) {
       subFields.put(fieldName, fieldDefinition);
-      hasAnalyzedField |= "analyzed".equals(fieldDefinition.get("index"));
       return this;
     }
 
@@ -277,36 +260,24 @@ public class NewIndex {
     }
 
     public void build() {
-      validate();
       Map<String, Object> hash = new TreeMap<>();
       if (subFields.isEmpty()) {
         hash.putAll(ImmutableMap.of(
           "type", "string",
           "index", disableSearch ? "no" : "not_analyzed",
-          "norms", ImmutableMap.of("enabled", String.valueOf(!disableNorms)),
-          "doc_values", docValues));
+          "norms", ImmutableMap.of("enabled", String.valueOf(!disableNorms))));
       } else {
         hash.put("type", "multi_field");
         Map<String, Object> multiFields = new TreeMap<>(subFields);
         multiFields.put(fieldName, ImmutableMap.of(
           "type", "string",
           "index", "not_analyzed",
-          "norms", ImmutableMap.of("enabled", "false"),
-          "doc_values", docValues));
+          "norms", ImmutableMap.of("enabled", "false")));
         hash.put("fields", multiFields);
       }
 
       indexType.setProperty(fieldName, hash);
     }
-
-    private void validate() {
-      if (docValues && hasAnalyzedField) {
-        throw new IllegalStateException("Doc values are not supported on analyzed strings of field: " + fieldName);
-      }
-      if (disableSearch && hasAnalyzedField) {
-        throw new IllegalStateException("Can't mix searchable and non-searchable arguments on field: " + fieldName);
-      }
-    }
   }
 
 

diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java
index f26a51b6431c26ef106fc9bf253fa294cf1b986c..5bd35dbf275aac15d7f42058a3ae6a5e8c9f72ba 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java
@@ -97,29 +97,29 @@ public class IssueIndexDefinition implements IndexDefinition {
     NewIndex.NewIndexType issueMapping = index.createType(TYPE_ISSUE);
     issueMapping.setAttribute("_parent", ImmutableMap.of("type", TYPE_AUTHORIZATION));
     issueMapping.setAttribute("_routing", ImmutableMap.of("required", "true"));
-    issueMapping.stringFieldBuilder(FIELD_ISSUE_ASSIGNEE).enableSorting().build();
-    issueMapping.stringFieldBuilder(FIELD_ISSUE_ATTRIBUTES).docValues().disableNorms().disableSearch().build();
-    issueMapping.stringFieldBuilder(FIELD_ISSUE_AUTHOR_LOGIN).disableNorms().docValues().build();
-    issueMapping.stringFieldBuilder(FIELD_ISSUE_COMPONENT_UUID).disableNorms().docValues().build();
+    issueMapping.stringFieldBuilder(FIELD_ISSUE_ASSIGNEE).disableNorms().enableSorting().build();
+    issueMapping.stringFieldBuilder(FIELD_ISSUE_ATTRIBUTES).disableNorms().disableSearch().build();
+    issueMapping.stringFieldBuilder(FIELD_ISSUE_AUTHOR_LOGIN).disableNorms().build();
+    issueMapping.stringFieldBuilder(FIELD_ISSUE_COMPONENT_UUID).disableNorms().build();
     issueMapping.createLongField(FIELD_ISSUE_EFFORT);
     issueMapping.createDoubleField(FIELD_ISSUE_GAP);
-    issueMapping.stringFieldBuilder(FIELD_ISSUE_FILE_PATH).enableSorting().build();
+    issueMapping.stringFieldBuilder(FIELD_ISSUE_FILE_PATH).disableNorms().enableSorting().build();
     issueMapping.createDateTimeField(FIELD_ISSUE_FUNC_CREATED_AT);
     issueMapping.createDateTimeField(FIELD_ISSUE_FUNC_UPDATED_AT);
     issueMapping.createDateTimeField(FIELD_ISSUE_FUNC_CLOSED_AT);
-    issueMapping.stringFieldBuilder(FIELD_ISSUE_KEY).enableSorting().build();
+    issueMapping.stringFieldBuilder(FIELD_ISSUE_KEY).disableNorms().enableSorting().build();
     issueMapping.stringFieldBuilder(FIELD_ISSUE_LANGUAGE).disableNorms().build();
     issueMapping.createIntegerField(FIELD_ISSUE_LINE);
-    issueMapping.stringFieldBuilder(FIELD_ISSUE_MESSAGE).disableNorms().docValues().build();
-    issueMapping.stringFieldBuilder(FIELD_ISSUE_MODULE_UUID).disableNorms().docValues().build();
+    issueMapping.stringFieldBuilder(FIELD_ISSUE_MESSAGE).disableNorms().build();
+    issueMapping.stringFieldBuilder(FIELD_ISSUE_MODULE_UUID).disableNorms().build();
     issueMapping.createUuidPathField(FIELD_ISSUE_MODULE_PATH);
     issueMapping.stringFieldBuilder(FIELD_ISSUE_PROJECT_UUID).disableNorms().enableSorting().build();
-    issueMapping.stringFieldBuilder(FIELD_ISSUE_DIRECTORY_PATH).disableNorms().docValues().build();
+    issueMapping.stringFieldBuilder(FIELD_ISSUE_DIRECTORY_PATH).disableNorms().build();
     issueMapping.stringFieldBuilder(FIELD_ISSUE_RESOLUTION).disableNorms().build();
     issueMapping.stringFieldBuilder(FIELD_ISSUE_RULE_KEY).disableNorms().build();
     issueMapping.stringFieldBuilder(FIELD_ISSUE_SEVERITY).disableNorms().build();
     issueMapping.createByteField(FIELD_ISSUE_SEVERITY_VALUE);
-    issueMapping.stringFieldBuilder(FIELD_ISSUE_STATUS).enableSorting().build();
+    issueMapping.stringFieldBuilder(FIELD_ISSUE_STATUS).disableNorms().enableSorting().build();
     issueMapping.stringFieldBuilder(FIELD_ISSUE_TAGS).disableNorms().build();
     issueMapping.createDateTimeField(FIELD_ISSUE_TECHNICAL_UPDATED_AT);
     issueMapping.stringFieldBuilder(FIELD_ISSUE_TYPE).disableNorms().build();

diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java
index ce9d2ac8d5dc86b2a88b2ad8ea02a04023b55c55..2c74c668d60496256a07d5dee20c92aa3ec5d3c9 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java
@@ -91,9 +91,9 @@ public class RuleIndexDefinition implements IndexDefinition {
     activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_KEY).enableSorting().build();
     activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_RULE_KEY).enableSorting().build();
     activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_REPOSITORY).build();
-    activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_PROFILE_KEY).disableNorms().docValues().build();
-    activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_INHERITANCE).disableNorms().docValues().build();
-    activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_SEVERITY).disableNorms().docValues().build();
+    activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_PROFILE_KEY).disableNorms().build();
+    activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_INHERITANCE).disableNorms().build();
+    activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_SEVERITY).disableNorms().build();
 
     activeRuleMapping.createLongField(RuleIndexDefinition.FIELD_ACTIVE_RULE_CREATED_AT);
     activeRuleMapping.createLongField(RuleIndexDefinition.FIELD_ACTIVE_RULE_UPDATED_AT);
@@ -105,22 +105,21 @@ public class RuleIndexDefinition implements IndexDefinition {
     ruleMapping.stringFieldBuilder(FIELD_RULE_KEY).enableSorting().build();
     ruleMapping.stringFieldBuilder(FIELD_RULE_RULE_KEY).enableSorting().build();
     ruleMapping.stringFieldBuilder(FIELD_RULE_REPOSITORY).build();
-    ruleMapping.stringFieldBuilder(FIELD_RULE_INTERNAL_KEY).disableNorms().disableSearch().docValues().build();
+    ruleMapping.stringFieldBuilder(FIELD_RULE_INTERNAL_KEY).disableNorms().disableSearch().build();
 
     ruleMapping.stringFieldBuilder(FIELD_RULE_NAME).enableSorting().enableWordSearch().build();
     ruleMapping.setProperty(FIELD_RULE_HTML_DESCRIPTION, ImmutableSortedMap.of(
       "type", "string",
       "index", "analyzed",
-      "doc_values", "false",
       "analyzer", "html_analyzer",
       "search_analyzer", "html_analyzer"
       ));
-    ruleMapping.stringFieldBuilder(FIELD_RULE_SEVERITY).disableNorms().docValues().build();
-    ruleMapping.stringFieldBuilder(FIELD_RULE_STATUS).disableNorms().docValues().build();
+    ruleMapping.stringFieldBuilder(FIELD_RULE_SEVERITY).disableNorms().build();
+    ruleMapping.stringFieldBuilder(FIELD_RULE_STATUS).disableNorms().build();
     ruleMapping.stringFieldBuilder(FIELD_RULE_LANGUAGE).disableNorms().build();
 
     ruleMapping.createBooleanField(FIELD_RULE_IS_TEMPLATE);
-    ruleMapping.stringFieldBuilder(FIELD_RULE_TEMPLATE_KEY).disableNorms().docValues().build();
+    ruleMapping.stringFieldBuilder(FIELD_RULE_TEMPLATE_KEY).disableNorms().build();
 
     ruleMapping.stringFieldBuilder(FIELD_RULE_ALL_TAGS).enableGramSearch().build();
     ruleMapping.stringFieldBuilder(FIELD_RULE_TYPE).disableNorms().build();

diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java
index d08875bfc0f66a93dcd01b3d0366f879ca1bc19d..17547dc0110aeebf202916c9db7563ab0f902891 100644 (file)
--- a/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java
@@ -112,40 +112,14 @@ public class NewIndexTest {
   }
 
   @Test
-  public void string_doc_values() {
+  public void use_default_doc_values() {
     NewIndex index = new NewIndex("issues");
     NewIndex.NewIndexType mapping = index.createType("issue");
-    mapping.stringFieldBuilder("the_doc_value").docValues().build();
+    mapping.stringFieldBuilder("the_doc_value").build();
 
     Map<String, Object> props = (Map) mapping.getProperty("the_doc_value");
     assertThat(props.get("type")).isEqualTo("string");
-    assertThat(props.get("index")).isEqualTo("not_analyzed");
-    assertThat(props.get("doc_values")).isEqualTo(Boolean.TRUE);
-    assertThat(props.get("fields")).isNull();
-  }
-
-  @Test
-  public void analyzed_strings_must_not_be_doc_values() {
-    NewIndex index = new NewIndex("issues");
-    NewIndex.NewIndexType mapping = index.createType("issue");
-    try {
-      mapping.stringFieldBuilder("the_doc_value").enableGramSearch().docValues().build();
-      fail();
-    } catch (IllegalStateException e) {
-      assertThat(e).hasMessage("Doc values are not supported on analyzed strings of field: the_doc_value");
-    }
-  }
-
-  @Test
-  public void do_not_disable_search_on_searchable_fields() {
-    NewIndex index = new NewIndex("issues");
-    NewIndex.NewIndexType mapping = index.createType("issue");
-    try {
-      mapping.stringFieldBuilder("my_field").enableGramSearch().disableSearch().build();
-      fail();
-    } catch (IllegalStateException e) {
-      assertThat(e).hasMessage("Can't mix searchable and non-searchable arguments on field: my_field");
-    }
+    assertThat(props.get("doc_values")).isNull();
   }
 
   @Test