Allow ES string fields with attribute "index: no"

Required to accept long string values on non-searchable fields, for example sourceLines/sourceLine/highlighting

4 files changed, 41 insertions, 10 deletions

diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java
index a3c0f87c525..4459fd5b3db 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java
@@ -123,7 +123,7 @@ public class NewIndex {
   public static class StringFieldBuilder {
     private final NewIndexType indexType;
     private final String fieldName;
-    private boolean sortable = false, wordSearch = false, gramSearch = false, docValues = false;
+    private boolean sortable = false, wordSearch = false, gramSearch = false, docValues = false, disableSearch = false;
 
     private StringFieldBuilder(NewIndexType indexType, String fieldName) {
       this.indexType = indexType;
@@ -159,12 +159,22 @@ public class NewIndex {
       return this;
     }
 
+    /**
+     * "index: no" -> Don’t index this field at all. This field will not be searchable.
+     * By default field is "not_analyzed": it is searchable, but index the value exactly
+     * as specified.
+     */
+    public StringFieldBuilder disableSearch() {
+      this.disableSearch = true;
+      return this;
+    }
+
     public void build() {
       validate();
-      Map<String, Object> hash = new TreeMap<String, Object>();
+      Map<String, Object> hash = new TreeMap<>();
       if (wordSearch || sortable || gramSearch) {
         hash.put("type", "multi_field");
-        Map<String, Object> multiFields = new TreeMap<String, Object>();
+        Map<String, Object> multiFields = new TreeMap<>();
 
         if (sortable) {
           multiFields.put(IndexField.SORT_SUFFIX, ImmutableSortedMap.of(
@@ -195,7 +205,7 @@ public class NewIndex {
       } else {
         hash.putAll(ImmutableMap.of(
           "type", "string",
-          "index", "not_analyzed",
+          "index", disableSearch ? "no" : "not_analyzed",
           "omit_norms", "true",
           "doc_values", docValues));
       }
@@ -205,7 +215,10 @@ public class NewIndex {
 
     private void validate() {
       if (docValues && (gramSearch || wordSearch || sortable)) {
-        throw new IllegalStateException("Doc values are not supported on analyzed strings");
+        throw new IllegalStateException("Doc values are not supported on analyzed strings of field: " + fieldName);
+      }
+      if (disableSearch && (gramSearch || wordSearch || sortable)) {
+        throw new IllegalStateException("Can't mix searchable and non-searchable arguments on field: " + fieldName);
       }
     }
   }
diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java
index a368fd142c9..440eef118cc 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java
@@ -114,7 +114,7 @@ public class IssueIndexDefinition implements IndexDefinition {
     issueMapping.setAttribute("_routing", ImmutableMap.of("required", true, "path", FIELD_ISSUE_PROJECT_UUID));
     issueMapping.stringFieldBuilder(FIELD_ISSUE_ACTION_PLAN).build();
     issueMapping.stringFieldBuilder(FIELD_ISSUE_ASSIGNEE).enableSorting().build();
-    issueMapping.stringFieldBuilder(FIELD_ISSUE_ATTRIBUTES).build();
+    issueMapping.stringFieldBuilder(FIELD_ISSUE_ATTRIBUTES).disableSearch().build();
     issueMapping.stringFieldBuilder(FIELD_ISSUE_AUTHOR_LOGIN).build();
     issueMapping.stringFieldBuilder(FIELD_ISSUE_COMPONENT_UUID).build();
     issueMapping.createLongField(FIELD_ISSUE_DEBT);
diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java
index 0b6a677dc7f..8b3f32cbdc5 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java
@@ -79,8 +79,8 @@ public class SourceLineIndexDefinition implements IndexDefinition {
     sourceLineMapping.stringFieldBuilder(FIELD_SCM_REVISION).build();
     sourceLineMapping.stringFieldBuilder(FIELD_SCM_AUTHOR).build();
     sourceLineMapping.createDateTimeField(FIELD_SCM_DATE);
-    sourceLineMapping.stringFieldBuilder(FIELD_HIGHLIGHTING).build();
-    sourceLineMapping.stringFieldBuilder(FIELD_SOURCE).build();
+    sourceLineMapping.stringFieldBuilder(FIELD_HIGHLIGHTING).disableSearch().build();
+    sourceLineMapping.stringFieldBuilder(FIELD_SOURCE).disableSearch().build();
     sourceLineMapping.createIntegerField(FIELD_UT_LINE_HITS);
     sourceLineMapping.createIntegerField(FIELD_UT_CONDITIONS);
     sourceLineMapping.createIntegerField(FIELD_UT_COVERED_CONDITIONS);
@@ -90,7 +90,7 @@ public class SourceLineIndexDefinition implements IndexDefinition {
     sourceLineMapping.createIntegerField(FIELD_OVERALL_LINE_HITS);
     sourceLineMapping.createIntegerField(FIELD_OVERALL_CONDITIONS);
     sourceLineMapping.createIntegerField(FIELD_OVERALL_COVERED_CONDITIONS);
-    sourceLineMapping.stringFieldBuilder(FIELD_SYMBOLS).build();
+    sourceLineMapping.stringFieldBuilder(FIELD_SYMBOLS).disableSearch().build();
     sourceLineMapping.createShortField(FIELD_DUPLICATIONS);
     sourceLineMapping.createDateTimeField(BaseNormalizer.UPDATED_AT_FIELD);
   }
diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java
index 5e6102c6108..28c7a00750a 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java
@@ -85,6 +85,7 @@ public class NewIndexTest {
     NewIndex index = new NewIndex("issues");
     NewIndex.NewIndexType mapping = index.createType("issue");
     mapping.stringFieldBuilder("basic_field").build();
+    mapping.stringFieldBuilder("not_searchable_field").disableSearch().build();
     mapping.stringFieldBuilder("all_capabilities_field")
       .enableGramSearch()
       .enableWordSearch()
@@ -96,6 +97,11 @@ public class NewIndexTest {
     assertThat(props.get("index")).isEqualTo("not_analyzed");
     assertThat(props.get("fields")).isNull();
 
+    props = (Map) mapping.getProperty("not_searchable_field");
+    assertThat(props.get("type")).isEqualTo("string");
+    assertThat(props.get("index")).isEqualTo("no");
+    assertThat(props.get("fields")).isNull();
+
     props = (Map) mapping.getProperty("all_capabilities_field");
     assertThat(props.get("type")).isEqualTo("multi_field");
     // no need to test values, it's not the scope of this test
@@ -123,7 +129,19 @@ public class NewIndexTest {
       mapping.stringFieldBuilder("the_doc_value").enableGramSearch().docValues().build();
       fail();
     } catch (IllegalStateException e) {
-      assertThat(e).hasMessage("Doc values are not supported on analyzed strings");
+      assertThat(e).hasMessage("Doc values are not supported on analyzed strings of field: the_doc_value");
+    }
+  }
+
+  @Test
+  public void do_not_disable_search_on_searchable_fields() throws Exception {
+    NewIndex index = new NewIndex("issues");
+    NewIndex.NewIndexType mapping = index.createType("issue");
+    try {
+      mapping.stringFieldBuilder("my_field").enableGramSearch().disableSearch().build();
+      fail();
+    } catch (IllegalStateException e) {
+      assertThat(e).hasMessage("Can't mix searchable and non-searchable arguments on field: my_field");
     }
   }
 }