SONAR-8798 support both text and keyword string field types

in addition, support fielddata for text type to allow sorting/aggregation/value retrieval

diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexDefinition.java
index 593334fb608700ae369eb9d800d687191d6415a7..2f77d3a5f5330694f7c79f7d7ea4fe37fc49713d 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexDefinition.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexDefinition.java
@@ -61,7 +61,8 @@ public class ComponentIndexDefinition implements IndexDefinition {
     mapping.keywordFieldBuilder(FIELD_UUID).disableNorms().build();
     mapping.keywordFieldBuilder(FIELD_PROJECT_UUID).disableNorms().build();
     mapping.keywordFieldBuilder(FIELD_KEY).addSubFields(SORTABLE_ANALYZER).build();
-    mapping.keywordFieldBuilder(FIELD_NAME)
+    mapping.textFieldBuilder(FIELD_NAME)
+      .withFieldData()
       .termVectorWithPositionOffsetsForAllSubfields()
       .addSubFields(NAME_ANALYZERS)
       .build();

diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettings.java b/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettings.java
index 24ca6110d7834ff373a8b234264d31057750632b..baf7959c3b4a76a7ffe41fff0c6fbe595ebe5db0 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettings.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettings.java
@@ -48,6 +48,8 @@ public class DefaultIndexSettings {
   public static final String INDEX_NOT_SEARCHABLE = "false";
   public static final String FIELD_TYPE_TEXT = "text";
   public static final String FIELD_TYPE_KEYWORD = "keyword";
+  public static final String FIELD_FIELDDATA = "fielddata";
+  public static final String FIELDDATA_ENABLED = "true";
   public static final String STANDARD = "standard";
   public static final String PATTERN = "pattern";
   public static final String CUSTOM = "custom";

diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettingsElement.java b/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettingsElement.java
index e8e3610d0fe6f9bc6e82d655d153a2a79bb146ed..4f2aac6aebde134a3465a5db63d8f5126d60f08f 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettingsElement.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettingsElement.java
@@ -30,6 +30,8 @@ import static org.sonar.server.es.DefaultIndexSettings.ANALYSIS;
 import static org.sonar.server.es.DefaultIndexSettings.ANALYZER;
 import static org.sonar.server.es.DefaultIndexSettings.ASCIIFOLDING;
 import static org.sonar.server.es.DefaultIndexSettings.DELIMITER;
+import static org.sonar.server.es.DefaultIndexSettings.FIELD_FIELDDATA;
+import static org.sonar.server.es.DefaultIndexSettings.FIELDDATA_ENABLED;
 import static org.sonar.server.es.DefaultIndexSettings.FIELD_TYPE_TEXT;
 import static org.sonar.server.es.DefaultIndexSettings.FILTER;
 import static org.sonar.server.es.DefaultIndexSettings.INDEX;
@@ -126,7 +128,8 @@ public enum DefaultIndexSettingsElement {
       return ImmutableSortedMap.of(
         TYPE, FIELD_TYPE_TEXT,
         INDEX, INDEX_SEARCHABLE,
-        ANALYZER, getName());
+        ANALYZER, getName(),
+        FIELD_FIELDDATA, FIELDDATA_ENABLED);
     }
   },
   INDEX_GRAMS_ANALYZER(ANALYZER) {

diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java
index de4e4377b9863adbfb38c805d690c48e607a591c..cc672e4345ab633d84dcd00cebd5551f9822c5f4 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java
@@ -39,6 +39,8 @@ import org.sonar.server.permission.index.AuthorizationTypeSupport;
 import static com.google.common.base.Preconditions.checkArgument;
 import static java.lang.String.format;
 import static org.sonar.server.es.DefaultIndexSettings.ANALYZER;
+import static org.sonar.server.es.DefaultIndexSettings.FIELDDATA_ENABLED;
+import static org.sonar.server.es.DefaultIndexSettings.FIELD_FIELDDATA;
 import static org.sonar.server.es.DefaultIndexSettings.FIELD_TYPE_KEYWORD;
 import static org.sonar.server.es.DefaultIndexSettings.FIELD_TYPE_TEXT;
 import static org.sonar.server.es.DefaultIndexSettings.INDEX;
@@ -144,6 +146,10 @@ public class NewIndex {
       return new KeywordFieldBuilder(this, fieldName);
     }
 
+    public TextFieldBuilder textFieldBuilder(String fieldName) {
+      return new TextFieldBuilder(this, fieldName);
+    }
+
     public NestedFieldBuilder nestedFieldBuilder(String fieldName) {
       return new NestedFieldBuilder(this, fieldName);
     }
@@ -196,7 +202,7 @@ public class NewIndex {
   /**
    * Helper to define a string field in mapping of index type
    */
-  public static class KeywordFieldBuilder {
+  public abstract static class StringFieldBuilder {
     private final NewIndexType indexType;
     private final String fieldName;
     private boolean disableSearch = false;
@@ -205,7 +211,7 @@ public class NewIndex {
     private SortedMap<String, Object> subFields = Maps.newTreeMap();
     private boolean store = false;
 
-    private KeywordFieldBuilder(NewIndexType indexType, String fieldName) {
+    private StringFieldBuilder(NewIndexType indexType, String fieldName) {
       this.indexType = indexType;
       this.fieldName = fieldName;
     }
@@ -214,7 +220,7 @@ public class NewIndex {
      * Add a sub-field. A {@code SortedMap} is required for consistency of the index settings hash.
      * @see IndexDefinitionHash
      */
-    private KeywordFieldBuilder addSubField(String fieldName, SortedMap<String, String> fieldDefinition) {
+    private StringFieldBuilder addSubField(String fieldName, SortedMap<String, String> fieldDefinition) {
       subFields.put(fieldName, fieldDefinition);
       return this;
     }
@@ -222,9 +228,9 @@ public class NewIndex {
     /**
      * Add subfields, one for each analyzer.
      */
-    public KeywordFieldBuilder addSubFields(DefaultIndexSettingsElement... analyzers) {
+    public StringFieldBuilder addSubFields(DefaultIndexSettingsElement... analyzers) {
       Arrays.stream(analyzers)
-          .forEach(analyzer -> addSubField(analyzer.getSubFieldSuffix(), analyzer.fieldMapping()));
+        .forEach(analyzer -> addSubField(analyzer.getSubFieldSuffix(), analyzer.fieldMapping()));
       return this;
     }
 
@@ -234,7 +240,7 @@ public class NewIndex {
      * https://www.elastic.co/guide/en/elasticsearch/reference/2.3/norms.html
      * https://www.elastic.co/guide/en/elasticsearch/guide/current/scoring-theory.html#field-norm
      */
-    public KeywordFieldBuilder disableNorms() {
+    public StringFieldBuilder disableNorms() {
       this.disableNorms = true;
       return this;
     }
@@ -242,7 +248,7 @@ public class NewIndex {
     /**
      * Position offset term vectors are required for the fast_vector_highlighter (fvh).
      */
-    public KeywordFieldBuilder termVectorWithPositionOffsetsForAllSubfields() {
+    public StringFieldBuilder termVectorWithPositionOffsetsForAllSubfields() {
       this.termVectorWithPositionOffsets = true;
       return this;
     }
@@ -252,12 +258,12 @@ public class NewIndex {
      * By default field is "true": it is searchable, but index the value exactly
      * as specified.
      */
-    public KeywordFieldBuilder disableSearch() {
+    public StringFieldBuilder disableSearch() {
       this.disableSearch = true;
       return this;
     }
 
-    public KeywordFieldBuilder store() {
+    public StringFieldBuilder store() {
       this.store = true;
       return this;
     }
@@ -266,12 +272,15 @@ public class NewIndex {
       Map<String, Object> hash = new TreeMap<>();
       if (subFields.isEmpty()) {
         hash.putAll(ImmutableMap.of(
-            "type", FIELD_TYPE_KEYWORD,
-            "index", disableSearch ? INDEX_NOT_SEARCHABLE : INDEX_SEARCHABLE,
-            "norms", String.valueOf(!disableNorms),
-            "store", String.valueOf(store)));
+          "type", getFieldType(),
+          "index", disableSearch ? INDEX_NOT_SEARCHABLE : INDEX_SEARCHABLE,
+          "norms", String.valueOf(!disableNorms),
+          "store", String.valueOf(store)));
+        if (getFieldData()) {
+          hash.put(FIELD_FIELDDATA, FIELDDATA_ENABLED);
+        }
       } else {
-        hash.put("type", "multi_field");
+        hash.put("type", getFieldType());
 
         Map<String, Object> multiFields = new TreeMap<>(subFields);
 
@@ -286,13 +295,24 @@ public class NewIndex {
             }
           });
         }
+        if (getFieldData()) {
+          multiFields.entrySet().forEach(entry -> {
+            Object subFieldMapping = entry.getValue();
+            if (subFieldMapping instanceof Map) {
+              entry.setValue(
+                addFieldToMapping(
+                  (Map<String, String>) subFieldMapping,
+                  FIELD_FIELDDATA, FIELDDATA_ENABLED));
+            }
+          });
+          hash.put(FIELD_FIELDDATA, FIELDDATA_ENABLED);
+        }
 
         multiFields.put(fieldName, ImmutableMap.of(
-            "type", FIELD_TYPE_KEYWORD,
-            "index", INDEX_SEARCHABLE,
-            "norms", "false",
-            "store", String.valueOf(store)
-        ));
+          "type", getFieldType(),
+          "index", INDEX_SEARCHABLE,
+          "norms", "false",
+          "store", String.valueOf(store)));
 
         hash.put("fields", multiFields);
       }
@@ -300,6 +320,10 @@ public class NewIndex {
       return indexType.setProperty(fieldName, hash);
     }
 
+    protected abstract boolean getFieldData();
+
+    protected abstract String getFieldType();
+
     private static SortedMap<String, String> addFieldToMapping(Map<String, String> source, String key, String value) {
       SortedMap<String, String> mutable = new TreeMap<>(source);
       mutable.put(key, value);
@@ -307,6 +331,49 @@ public class NewIndex {
     }
   }
 
+  public static class KeywordFieldBuilder extends StringFieldBuilder {
+
+    private KeywordFieldBuilder(NewIndexType indexType, String fieldName) {
+      super(indexType, fieldName);
+    }
+
+    @Override
+    protected boolean getFieldData() {
+      return false;
+    }
+
+    protected String getFieldType() {
+      return FIELD_TYPE_KEYWORD;
+    }
+  }
+
+  public static class TextFieldBuilder extends StringFieldBuilder {
+
+    private boolean fieldData = false;
+
+    private TextFieldBuilder(NewIndexType indexType, String fieldName) {
+      super(indexType, fieldName);
+    }
+
+    protected String getFieldType() {
+      return FIELD_TYPE_TEXT;
+    }
+
+    /**
+     * Required to enable sorting, aggregation and access to field data on fields of type "text".
+     * <p>Disabled by default as this can have significant memory cost</p>
+     */
+    public StringFieldBuilder withFieldData() {
+      this.fieldData = true;
+      return this;
+    }
+
+    @Override
+    protected boolean getFieldData() {
+      return fieldData;
+    }
+  }
+
   public static class NestedFieldBuilder {
     private final NewIndexType indexType;
     private final String fieldName;