From 627a14b11aed4a2df73342d10ed8938adc58f167 Mon Sep 17 00:00:00 2001 From: Simon Brandhof Date: Tue, 10 Feb 2015 17:57:09 +0100 Subject: [PATCH] Allow ES string fields with attribute "index: no" Required to accept long string values on non-searchable fields, for example sourceLines/sourceLine/highlighting --- .../java/org/sonar/server/es/NewIndex.java | 23 +++++++++++++++---- .../issue/index/IssueIndexDefinition.java | 2 +- .../index/SourceLineIndexDefinition.java | 6 ++--- .../org/sonar/server/es/NewIndexTest.java | 20 +++++++++++++++- 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java index a3c0f87c525..4459fd5b3db 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java @@ -123,7 +123,7 @@ public class NewIndex { public static class StringFieldBuilder { private final NewIndexType indexType; private final String fieldName; - private boolean sortable = false, wordSearch = false, gramSearch = false, docValues = false; + private boolean sortable = false, wordSearch = false, gramSearch = false, docValues = false, disableSearch = false; private StringFieldBuilder(NewIndexType indexType, String fieldName) { this.indexType = indexType; @@ -159,12 +159,22 @@ public class NewIndex { return this; } + /** + * "index: no" -> Don’t index this field at all. This field will not be searchable. + * By default field is "not_analyzed": it is searchable, but index the value exactly + * as specified. + */ + public StringFieldBuilder disableSearch() { + this.disableSearch = true; + return this; + } + public void build() { validate(); - Map hash = new TreeMap(); + Map hash = new TreeMap<>(); if (wordSearch || sortable || gramSearch) { hash.put("type", "multi_field"); - Map multiFields = new TreeMap(); + Map multiFields = new TreeMap<>(); if (sortable) { multiFields.put(IndexField.SORT_SUFFIX, ImmutableSortedMap.of( @@ -195,7 +205,7 @@ public class NewIndex { } else { hash.putAll(ImmutableMap.of( "type", "string", - "index", "not_analyzed", + "index", disableSearch ? "no" : "not_analyzed", "omit_norms", "true", "doc_values", docValues)); } @@ -205,7 +215,10 @@ public class NewIndex { private void validate() { if (docValues && (gramSearch || wordSearch || sortable)) { - throw new IllegalStateException("Doc values are not supported on analyzed strings"); + throw new IllegalStateException("Doc values are not supported on analyzed strings of field: " + fieldName); + } + if (disableSearch && (gramSearch || wordSearch || sortable)) { + throw new IllegalStateException("Can't mix searchable and non-searchable arguments on field: " + fieldName); } } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java index a368fd142c9..440eef118cc 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java @@ -114,7 +114,7 @@ public class IssueIndexDefinition implements IndexDefinition { issueMapping.setAttribute("_routing", ImmutableMap.of("required", true, "path", FIELD_ISSUE_PROJECT_UUID)); issueMapping.stringFieldBuilder(FIELD_ISSUE_ACTION_PLAN).build(); issueMapping.stringFieldBuilder(FIELD_ISSUE_ASSIGNEE).enableSorting().build(); - issueMapping.stringFieldBuilder(FIELD_ISSUE_ATTRIBUTES).build(); + issueMapping.stringFieldBuilder(FIELD_ISSUE_ATTRIBUTES).disableSearch().build(); issueMapping.stringFieldBuilder(FIELD_ISSUE_AUTHOR_LOGIN).build(); issueMapping.stringFieldBuilder(FIELD_ISSUE_COMPONENT_UUID).build(); issueMapping.createLongField(FIELD_ISSUE_DEBT); diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java index 0b6a677dc7f..8b3f32cbdc5 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java @@ -79,8 +79,8 @@ public class SourceLineIndexDefinition implements IndexDefinition { sourceLineMapping.stringFieldBuilder(FIELD_SCM_REVISION).build(); sourceLineMapping.stringFieldBuilder(FIELD_SCM_AUTHOR).build(); sourceLineMapping.createDateTimeField(FIELD_SCM_DATE); - sourceLineMapping.stringFieldBuilder(FIELD_HIGHLIGHTING).build(); - sourceLineMapping.stringFieldBuilder(FIELD_SOURCE).build(); + sourceLineMapping.stringFieldBuilder(FIELD_HIGHLIGHTING).disableSearch().build(); + sourceLineMapping.stringFieldBuilder(FIELD_SOURCE).disableSearch().build(); sourceLineMapping.createIntegerField(FIELD_UT_LINE_HITS); sourceLineMapping.createIntegerField(FIELD_UT_CONDITIONS); sourceLineMapping.createIntegerField(FIELD_UT_COVERED_CONDITIONS); @@ -90,7 +90,7 @@ public class SourceLineIndexDefinition implements IndexDefinition { sourceLineMapping.createIntegerField(FIELD_OVERALL_LINE_HITS); sourceLineMapping.createIntegerField(FIELD_OVERALL_CONDITIONS); sourceLineMapping.createIntegerField(FIELD_OVERALL_COVERED_CONDITIONS); - sourceLineMapping.stringFieldBuilder(FIELD_SYMBOLS).build(); + sourceLineMapping.stringFieldBuilder(FIELD_SYMBOLS).disableSearch().build(); sourceLineMapping.createShortField(FIELD_DUPLICATIONS); sourceLineMapping.createDateTimeField(BaseNormalizer.UPDATED_AT_FIELD); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java index 5e6102c6108..28c7a00750a 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java @@ -85,6 +85,7 @@ public class NewIndexTest { NewIndex index = new NewIndex("issues"); NewIndex.NewIndexType mapping = index.createType("issue"); mapping.stringFieldBuilder("basic_field").build(); + mapping.stringFieldBuilder("not_searchable_field").disableSearch().build(); mapping.stringFieldBuilder("all_capabilities_field") .enableGramSearch() .enableWordSearch() @@ -96,6 +97,11 @@ public class NewIndexTest { assertThat(props.get("index")).isEqualTo("not_analyzed"); assertThat(props.get("fields")).isNull(); + props = (Map) mapping.getProperty("not_searchable_field"); + assertThat(props.get("type")).isEqualTo("string"); + assertThat(props.get("index")).isEqualTo("no"); + assertThat(props.get("fields")).isNull(); + props = (Map) mapping.getProperty("all_capabilities_field"); assertThat(props.get("type")).isEqualTo("multi_field"); // no need to test values, it's not the scope of this test @@ -123,7 +129,19 @@ public class NewIndexTest { mapping.stringFieldBuilder("the_doc_value").enableGramSearch().docValues().build(); fail(); } catch (IllegalStateException e) { - assertThat(e).hasMessage("Doc values are not supported on analyzed strings"); + assertThat(e).hasMessage("Doc values are not supported on analyzed strings of field: the_doc_value"); + } + } + + @Test + public void do_not_disable_search_on_searchable_fields() throws Exception { + NewIndex index = new NewIndex("issues"); + NewIndex.NewIndexType mapping = index.createType("issue"); + try { + mapping.stringFieldBuilder("my_field").enableGramSearch().disableSearch().build(); + fail(); + } catch (IllegalStateException e) { + assertThat(e).hasMessage("Can't mix searchable and non-searchable arguments on field: my_field"); } } } -- 2.39.5