From 84100b887e2a6abb97cc6db752dbef39e6994c25 Mon Sep 17 00:00:00 2001 From: =?utf8?q?S=C3=A9bastien=20Lesaint?= Date: Thu, 7 Sep 2017 17:31:01 +0200 Subject: [PATCH] SONAR-9796 add KeywordFieldBuilder#disableSortingAndAggregating --- .../java/org/sonar/server/es/NewIndex.java | 77 ++++++++++++------- .../org/sonar/server/es/NewIndexTest.java | 20 ++++- 2 files changed, 68 insertions(+), 29 deletions(-) diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java index bba7ef58450..36ed5c3fff7 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java @@ -271,7 +271,7 @@ public class NewIndex { /** * Helper to define a string field in mapping of index type */ - public abstract static class StringFieldBuilder { + public abstract static class StringFieldBuilder> { private final NewIndexType indexType; private final String fieldName; private boolean disableSearch = false; @@ -279,6 +279,7 @@ public class NewIndex { private boolean termVectorWithPositionOffsets = false; private SortedMap subFields = Maps.newTreeMap(); private boolean store = false; + protected boolean disabledDocValues = false; private StringFieldBuilder(NewIndexType indexType, String fieldName) { this.indexType = indexType; @@ -289,18 +290,18 @@ public class NewIndex { * Add a sub-field. A {@code SortedMap} is required for consistency of the index settings hash. * @see IndexDefinitionHash */ - private StringFieldBuilder addSubField(String fieldName, SortedMap fieldDefinition) { + private T addSubField(String fieldName, SortedMap fieldDefinition) { subFields.put(fieldName, fieldDefinition); - return this; + return castThis(); } /** * Add subfields, one for each analyzer. */ - public StringFieldBuilder addSubFields(DefaultIndexSettingsElement... analyzers) { + public T addSubFields(DefaultIndexSettingsElement... analyzers) { Arrays.stream(analyzers) .forEach(analyzer -> addSubField(analyzer.getSubFieldSuffix(), analyzer.fieldMapping())); - return this; + return castThis(); } /** @@ -309,17 +310,17 @@ public class NewIndex { * https://www.elastic.co/guide/en/elasticsearch/reference/2.3/norms.html * https://www.elastic.co/guide/en/elasticsearch/guide/current/scoring-theory.html#field-norm */ - public StringFieldBuilder disableNorms() { + public T disableNorms() { this.disableNorms = true; - return this; + return castThis(); } /** * Position offset term vectors are required for the fast_vector_highlighter (fvh). */ - public StringFieldBuilder termVectorWithPositionOffsets() { + public T termVectorWithPositionOffsets() { this.termVectorWithPositionOffsets = true; - return this; + return castThis(); } /** @@ -327,14 +328,19 @@ public class NewIndex { * By default field is "true": it is searchable, but index the value exactly * as specified. */ - public StringFieldBuilder disableSearch() { + public T disableSearch() { this.disableSearch = true; - return this; + return castThis(); } - public StringFieldBuilder store() { + public T store() { this.store = true; - return this; + return castThis(); + } + + @SuppressWarnings("unchecked") + private T castThis() { + return (T) this; } public NewIndexType build() { @@ -345,16 +351,18 @@ public class NewIndex { } private NewIndexType buildWithoutSubfields() { - Map hash = new TreeMap<>(); - hash.putAll(ImmutableMap.of( - "type", getFieldType(), - INDEX, disableSearch ? INDEX_NOT_SEARCHABLE : INDEX_SEARCHABLE, - "norms", valueOf(!disableNorms), - "store", valueOf(store))); + ImmutableMap.Builder hash = ImmutableMap.builder(); + hash.put("type", getFieldType()) + .put(INDEX, disableSearch ? INDEX_NOT_SEARCHABLE : INDEX_SEARCHABLE) + .put("norms", valueOf(!disableNorms)) + .put("store", valueOf(store)); + if (FIELD_TYPE_KEYWORD.equals(getFieldType())) { + hash.put("doc_values", valueOf(!disabledDocValues)); + } if (getFieldData()) { hash.put(FIELD_FIELDDATA, FIELDDATA_ENABLED); } - return indexType.setProperty(fieldName, hash); + return indexType.setProperty(fieldName, hash.build()); } private NewIndexType buildWithSubfields() { @@ -388,11 +396,15 @@ public class NewIndex { hash.put(FIELD_FIELDDATA, FIELDDATA_ENABLED); } - multiFields.put(fieldName, ImmutableMap.of( - "type", getFieldType(), - INDEX, INDEX_SEARCHABLE, - "norms", "false", - "store", valueOf(store))); + ImmutableMap.Builder builder = ImmutableMap.builder(); + builder.put("type", getFieldType()) + .put(INDEX, INDEX_SEARCHABLE) + .put("norms", "false") + .put("store", valueOf(store)); + if (FIELD_TYPE_KEYWORD.equals(getFieldType())) { + builder.put("doc_values", valueOf(!disabledDocValues)); + } + multiFields.put(fieldName, builder.build()); hash.put("fields", multiFields); return indexType.setProperty(fieldName, hash); @@ -409,7 +421,7 @@ public class NewIndex { } } - public static class KeywordFieldBuilder extends StringFieldBuilder { + public static class KeywordFieldBuilder extends StringFieldBuilder { private KeywordFieldBuilder(NewIndexType indexType, String fieldName) { super(indexType, fieldName); @@ -423,9 +435,20 @@ public class NewIndex { protected String getFieldType() { return FIELD_TYPE_KEYWORD; } + + /** + * By default, field is stored on disk in a column-stride fashion, so that it can later be used for sorting, + * aggregations, or scripting. + * Disabling this reduces the size of the index and drop the constraint of single term max size of + * 32766 bytes (which, if there is no tokenizing enabled on the field, equals the size of the whole data). + */ + public KeywordFieldBuilder disableSortingAndAggregating() { + this.disabledDocValues = true; + return this; + } } - public static class TextFieldBuilder extends StringFieldBuilder { + public static class TextFieldBuilder extends StringFieldBuilder { private boolean fieldData = false; diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java index a3dcf87c1df..794d13c28b7 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java @@ -128,6 +128,11 @@ public class NewIndexTest { DefaultIndexSettingsElement.SEARCH_WORDS_ANALYZER, DefaultIndexSettingsElement.SORTABLE_ANALYZER) .build(); + mapping.keywordFieldBuilder("dumb_text_storage") + .disableSearch() + .disableNorms() + .disableSortingAndAggregating() + .build(); Map props = (Map) mapping.getProperty("basic_field"); assertThat(props.get("type")).isEqualTo("keyword"); @@ -137,12 +142,23 @@ public class NewIndexTest { props = (Map) mapping.getProperty("not_searchable_field"); assertThat(props.get("type")).isEqualTo("keyword"); assertThat(props.get("index")).isEqualTo("false"); + assertThat(props.get("norms")).isEqualTo("true"); + assertThat(props.get("store")).isEqualTo("false"); + assertThat(props.get("doc_values")).isEqualTo("true"); assertThat(props.get("fields")).isNull(); props = (Map) mapping.getProperty("all_capabilities_field"); assertThat(props.get("type")).isEqualTo("keyword"); // no need to test values, it's not the scope of this test assertThat((Map) props.get("fields")).isNotEmpty(); + + props = (Map) mapping.getProperty("dumb_text_storage"); + assertThat(props.get("type")).isEqualTo("keyword"); + assertThat(props.get("index")).isEqualTo("false"); + assertThat(props.get("norms")).isEqualTo("false"); + assertThat(props.get("store")).isEqualTo("false"); + assertThat(props.get("doc_values")).isEqualTo("false"); + assertThat(props.get("fields")).isNull(); } @Test @@ -174,14 +190,14 @@ public class NewIndexTest { } @Test - public void use_default_doc_values() { + public void use_doc_values_by_default() { NewIndex index = new NewIndex("issues", defaultSettingsConfiguration); NewIndex.NewIndexType mapping = index.createType("issue"); mapping.keywordFieldBuilder("the_doc_value").build(); Map props = (Map) mapping.getProperty("the_doc_value"); assertThat(props.get("type")).isEqualTo("keyword"); - assertThat(props.get("doc_values")).isNull(); + assertThat(props.get("doc_values")).isEqualTo("true"); } @Test -- 2.39.5