SONAR-7645 Disable ES norms when possible

Norms are used when a string field is involved
in a match query. Basically they are useless
when only filters or aggregations are executed.
In this case they can be disabled. It won't
consume memory for nothing.

diff --git a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexDefinition.java
index be4f363d486ed2058c2a9b04c54c3fc7a2c8a0f9..0ded443ffde75308848cbc3bac0527a33369112b 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexDefinition.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexDefinition.java
@@ -55,12 +55,12 @@ public class ActivityIndexDefinition implements IndexDefinition {
     // type "activity"
     NewIndex.NewIndexType mapping = index.createType(TYPE);
     mapping.setAttribute("_id", ImmutableMap.of("path", FIELD_KEY));
-    mapping.stringFieldBuilder(FIELD_KEY).build();
-    mapping.stringFieldBuilder(FIELD_TYPE).build();
-    mapping.stringFieldBuilder(FIELD_ACTION).build();
-    mapping.stringFieldBuilder(FIELD_LOGIN).build();
+    mapping.stringFieldBuilder(FIELD_KEY).disableNorms().build();
+    mapping.stringFieldBuilder(FIELD_TYPE).disableNorms().build();
+    mapping.stringFieldBuilder(FIELD_ACTION).disableNorms().build();
+    mapping.stringFieldBuilder(FIELD_LOGIN).disableNorms().build();
     mapping.createDynamicNestedField(FIELD_DETAILS);
-    mapping.stringFieldBuilder(FIELD_MESSAGE).build();
+    mapping.stringFieldBuilder(FIELD_MESSAGE).disableNorms().build();
     mapping.createDateTimeField(FIELD_CREATED_AT);
   }
 }

diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java
index 668f838df7daea77e225ec6bd269b7489b554a0f..dfb01c8ff07f752c4f93fef27ac6b97efba6dd30 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java
@@ -168,6 +168,7 @@ public class NewIndex {
     private boolean docValues = false;
     private boolean disableSearch = false;
     private boolean hasAnalyzedField = false;
+    private boolean disableNorms = false;
     private SortedMap<String, Object> subFields = Maps.newTreeMap();
 
     private StringFieldBuilder(NewIndexType indexType, String fieldName) {
@@ -234,6 +235,17 @@ public class NewIndex {
       return this;
     }
 
+    /**
+     * Norms consume useless memory if string field is used for filtering or aggregations.
+     *
+     * https://www.elastic.co/guide/en/elasticsearch/reference/2.3/norms.html
+     * https://www.elastic.co/guide/en/elasticsearch/guide/current/scoring-theory.html#field-norm
+     */
+    public StringFieldBuilder disableNorms() {
+      this.disableNorms = true;
+      return this;
+    }
+
     /**
      * "index: no" -> Don’t index this field at all. This field will not be searchable.
      * By default field is "not_analyzed": it is searchable, but index the value exactly
@@ -251,7 +263,7 @@ public class NewIndex {
         hash.putAll(ImmutableMap.of(
           "type", "string",
           "index", disableSearch ? "no" : "not_analyzed",
-          "omit_norms", "true",
+          "norms", ImmutableMap.of("enabled", String.valueOf(!disableNorms)),
           "doc_values", docValues));
       } else {
         hash.put("type", "multi_field");
@@ -259,7 +271,7 @@ public class NewIndex {
         multiFields.put(fieldName, ImmutableMap.of(
           "type", "string",
           "index", "not_analyzed",
-          "omit_norms", "true",
+          "norms", ImmutableMap.of("enabled", "false"),
           "doc_values", docValues));
         hash.put("fields", multiFields);
       }

diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java
index 0739bda24337aee9095f9fe775999a176fc2e694..e5f0e19578af975ace44e43800d7ba5d8319a416 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java
@@ -695,15 +695,6 @@ public class IssueIndex extends BaseIndex {
     return boolFilter;
   }
 
-  /**
-   * TODO used only by tests, so must be replaced by EsTester#countDocuments()
-   */
-  public long countAll() {
-    return getClient().prepareCount(IssueIndexDefinition.INDEX)
-      .setTypes(IssueIndexDefinition.TYPE_ISSUE)
-      .get().getCount();
-  }
-
   /**
    * Return non closed issues for a given project, module, or file. Other kind of components are not allowed.
    * Only fields needed for the batch are returned.

diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java
index 9194033d44787fd09ffe350ffe7447a1142840e6..d0f4ccc40b908c7e35c174a7b7e7dce2d8d59bc5 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java
@@ -97,9 +97,9 @@ public class IssueIndexDefinition implements IndexDefinition {
     NewIndex.NewIndexType authorizationMapping = index.createType(TYPE_AUTHORIZATION);
     authorizationMapping.setAttribute("_id", ImmutableMap.of("path", FIELD_AUTHORIZATION_PROJECT_UUID));
     authorizationMapping.createDateTimeField(FIELD_AUTHORIZATION_UPDATED_AT);
-    authorizationMapping.stringFieldBuilder(FIELD_AUTHORIZATION_PROJECT_UUID).build();
-    authorizationMapping.stringFieldBuilder(FIELD_AUTHORIZATION_GROUPS).build();
-    authorizationMapping.stringFieldBuilder(FIELD_AUTHORIZATION_USERS).build();
+    authorizationMapping.stringFieldBuilder(FIELD_AUTHORIZATION_PROJECT_UUID).disableNorms().build();
+    authorizationMapping.stringFieldBuilder(FIELD_AUTHORIZATION_GROUPS).disableNorms().build();
+    authorizationMapping.stringFieldBuilder(FIELD_AUTHORIZATION_USERS).disableNorms().build();
 
     // type "issue"
     NewIndex.NewIndexType issueMapping = index.createType(TYPE_ISSUE);
@@ -107,9 +107,9 @@ public class IssueIndexDefinition implements IndexDefinition {
     issueMapping.setAttribute("_parent", ImmutableMap.of("type", TYPE_AUTHORIZATION));
     issueMapping.setAttribute("_routing", ImmutableMap.of("required", true, "path", FIELD_ISSUE_PROJECT_UUID));
     issueMapping.stringFieldBuilder(FIELD_ISSUE_ASSIGNEE).enableSorting().build();
-    issueMapping.stringFieldBuilder(FIELD_ISSUE_ATTRIBUTES).docValues().disableSearch().build();
-    issueMapping.stringFieldBuilder(FIELD_ISSUE_AUTHOR_LOGIN).docValues().build();
-    issueMapping.stringFieldBuilder(FIELD_ISSUE_COMPONENT_UUID).docValues().build();
+    issueMapping.stringFieldBuilder(FIELD_ISSUE_ATTRIBUTES).docValues().disableNorms().disableSearch().build();
+    issueMapping.stringFieldBuilder(FIELD_ISSUE_AUTHOR_LOGIN).disableNorms().docValues().build();
+    issueMapping.stringFieldBuilder(FIELD_ISSUE_COMPONENT_UUID).disableNorms().docValues().build();
     issueMapping.createLongField(FIELD_ISSUE_EFFORT);
     issueMapping.createDoubleField(FIELD_ISSUE_GAP);
     issueMapping.stringFieldBuilder(FIELD_ISSUE_FILE_PATH).enableSorting().build();
@@ -117,20 +117,20 @@ public class IssueIndexDefinition implements IndexDefinition {
     issueMapping.createDateTimeField(FIELD_ISSUE_FUNC_UPDATED_AT);
     issueMapping.createDateTimeField(FIELD_ISSUE_FUNC_CLOSED_AT);
     issueMapping.stringFieldBuilder(FIELD_ISSUE_KEY).enableSorting().build();
-    issueMapping.stringFieldBuilder(FIELD_ISSUE_LANGUAGE).build();
+    issueMapping.stringFieldBuilder(FIELD_ISSUE_LANGUAGE).disableNorms().build();
     issueMapping.createIntegerField(FIELD_ISSUE_LINE);
-    issueMapping.stringFieldBuilder(FIELD_ISSUE_MESSAGE).docValues().build();
-    issueMapping.stringFieldBuilder(FIELD_ISSUE_MODULE_UUID).docValues().build();
+    issueMapping.stringFieldBuilder(FIELD_ISSUE_MESSAGE).disableNorms().docValues().build();
+    issueMapping.stringFieldBuilder(FIELD_ISSUE_MODULE_UUID).disableNorms().docValues().build();
     issueMapping.createUuidPathField(FIELD_ISSUE_MODULE_PATH);
-    issueMapping.stringFieldBuilder(FIELD_ISSUE_PROJECT_UUID).enableSorting().build();
-    issueMapping.stringFieldBuilder(FIELD_ISSUE_DIRECTORY_PATH).docValues().build();
-    issueMapping.stringFieldBuilder(FIELD_ISSUE_RESOLUTION).build();
-    issueMapping.stringFieldBuilder(FIELD_ISSUE_RULE_KEY).build();
-    issueMapping.stringFieldBuilder(FIELD_ISSUE_SEVERITY).build();
+    issueMapping.stringFieldBuilder(FIELD_ISSUE_PROJECT_UUID).disableNorms().enableSorting().build();
+    issueMapping.stringFieldBuilder(FIELD_ISSUE_DIRECTORY_PATH).disableNorms().docValues().build();
+    issueMapping.stringFieldBuilder(FIELD_ISSUE_RESOLUTION).disableNorms().build();
+    issueMapping.stringFieldBuilder(FIELD_ISSUE_RULE_KEY).disableNorms().build();
+    issueMapping.stringFieldBuilder(FIELD_ISSUE_SEVERITY).disableNorms().build();
     issueMapping.createByteField(FIELD_ISSUE_SEVERITY_VALUE);
     issueMapping.stringFieldBuilder(FIELD_ISSUE_STATUS).enableSorting().build();
-    issueMapping.stringFieldBuilder(FIELD_ISSUE_TAGS).build();
+    issueMapping.stringFieldBuilder(FIELD_ISSUE_TAGS).disableNorms().build();
     issueMapping.createDateTimeField(FIELD_ISSUE_TECHNICAL_UPDATED_AT);
-    issueMapping.stringFieldBuilder(FIELD_ISSUE_TYPE).build();
+    issueMapping.stringFieldBuilder(FIELD_ISSUE_TYPE).disableNorms().build();
   }
 }

diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java
index 5b5db55752bc0ddea6680c46e373f01e6054135d..19be74edf104cd14cc929a274a1fc0b16a59da59 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java
@@ -91,8 +91,8 @@ public class RuleIndexDefinition implements IndexDefinition {
 
     ruleMapping.stringFieldBuilder(FIELD_RULE_KEY).enableSorting().build();
     ruleMapping.stringFieldBuilder(FIELD_RULE_RULE_KEY).enableSorting().build();
-    ruleMapping.stringFieldBuilder(FIELD_RULE_REPOSITORY).docValues().build();
-    ruleMapping.stringFieldBuilder(FIELD_RULE_INTERNAL_KEY).disableSearch().docValues().build();
+    ruleMapping.stringFieldBuilder(FIELD_RULE_REPOSITORY).disableNorms().docValues().build();
+    ruleMapping.stringFieldBuilder(FIELD_RULE_INTERNAL_KEY).disableNorms().disableSearch().docValues().build();
 
     ruleMapping.stringFieldBuilder(FIELD_RULE_NAME).enableSorting().enableWordSearch().build();
     ruleMapping.setProperty(FIELD_RULE_HTML_DESCRIPTION, ImmutableSortedMap.of(
@@ -102,15 +102,15 @@ public class RuleIndexDefinition implements IndexDefinition {
       "index_analyzer", "html_analyzer",
       "search_analyzer", "html_analyzer"
       ));
-    ruleMapping.stringFieldBuilder(FIELD_RULE_SEVERITY).docValues().build();
-    ruleMapping.stringFieldBuilder(FIELD_RULE_STATUS).docValues().build();
-    ruleMapping.stringFieldBuilder(FIELD_RULE_LANGUAGE).build();
+    ruleMapping.stringFieldBuilder(FIELD_RULE_SEVERITY).disableNorms().docValues().build();
+    ruleMapping.stringFieldBuilder(FIELD_RULE_STATUS).disableNorms().docValues().build();
+    ruleMapping.stringFieldBuilder(FIELD_RULE_LANGUAGE).disableNorms().build();
 
     ruleMapping.createBooleanField(FIELD_RULE_IS_TEMPLATE);
-    ruleMapping.stringFieldBuilder(FIELD_RULE_TEMPLATE_KEY).docValues().build();
+    ruleMapping.stringFieldBuilder(FIELD_RULE_TEMPLATE_KEY).disableNorms().docValues().build();
 
     ruleMapping.stringFieldBuilder(FIELD_RULE_ALL_TAGS).enableGramSearch().build();
-    ruleMapping.stringFieldBuilder(FIELD_RULE_TYPE).build();
+    ruleMapping.stringFieldBuilder(FIELD_RULE_TYPE).disableNorms().build();
 
     ruleMapping.createLongField(FIELD_RULE_CREATED_AT);
     ruleMapping.createLongField(FIELD_RULE_UPDATED_AT);
@@ -123,11 +123,11 @@ public class RuleIndexDefinition implements IndexDefinition {
     activeRuleMapping.setAttribute("_routing", ImmutableMap.of("required", true, "path", RuleIndexDefinition.FIELD_ACTIVE_RULE_REPOSITORY));
 
     activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_KEY).enableSorting().build();
-    activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_RULE_KEY).disableSearch().docValues().build();
-    activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_REPOSITORY).disableSearch().docValues().build();
-    activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_PROFILE_KEY).docValues().build();
-    activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_INHERITANCE).docValues().build();
-    activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_SEVERITY).docValues().build();
+    activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_RULE_KEY).disableNorms().disableSearch().docValues().build();
+    activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_REPOSITORY).disableNorms().disableSearch().docValues().build();
+    activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_PROFILE_KEY).disableNorms().docValues().build();
+    activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_INHERITANCE).disableNorms().docValues().build();
+    activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_SEVERITY).disableNorms().docValues().build();
 
     activeRuleMapping.createLongField(RuleIndexDefinition.FIELD_ACTIVE_RULE_CREATED_AT);
     activeRuleMapping.createLongField(RuleIndexDefinition.FIELD_ACTIVE_RULE_UPDATED_AT);

diff --git a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexDefinition.java
index eddab9581edcf277b71571abeba6cebb58e6a29d..6b4c532c4f87ee40e096518f2bf403f9070bbb6f 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexDefinition.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexDefinition.java
@@ -60,14 +60,14 @@ public class TestIndexDefinition implements IndexDefinition {
     NewIndex.NewIndexType mapping = index.createType(TYPE);
     mapping.setAttribute("_id", ImmutableMap.of("path", FIELD_TEST_UUID));
     mapping.setAttribute("_routing", ImmutableMap.of("required", true, "path", FIELD_PROJECT_UUID));
-    mapping.stringFieldBuilder(FIELD_PROJECT_UUID).build();
-    mapping.stringFieldBuilder(FIELD_FILE_UUID).build();
-    mapping.stringFieldBuilder(FIELD_TEST_UUID).build();
-    mapping.stringFieldBuilder(FIELD_NAME).disableSearch().build();
-    mapping.stringFieldBuilder(FIELD_STATUS).disableSearch().build();
+    mapping.stringFieldBuilder(FIELD_PROJECT_UUID).disableNorms().build();
+    mapping.stringFieldBuilder(FIELD_FILE_UUID).disableNorms().build();
+    mapping.stringFieldBuilder(FIELD_TEST_UUID).disableNorms().build();
+    mapping.stringFieldBuilder(FIELD_NAME).disableNorms().disableSearch().build();
+    mapping.stringFieldBuilder(FIELD_STATUS).disableNorms().disableSearch().build();
     mapping.createLongField(FIELD_DURATION_IN_MS);
-    mapping.stringFieldBuilder(FIELD_MESSAGE).disableSearch().build();
-    mapping.stringFieldBuilder(FIELD_STACKTRACE).disableSearch().build();
+    mapping.stringFieldBuilder(FIELD_MESSAGE).disableNorms().disableSearch().build();
+    mapping.stringFieldBuilder(FIELD_STACKTRACE).disableNorms().disableSearch().build();
     mapping.nestedObjectBuilder(FIELD_COVERED_FILES, nestedMapping).build();
     mapping.createDateTimeField(FIELD_UPDATED_AT);
   }

diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexDefinition.java
index 0b5600b34b1626da2a860a9ee04e30cb8d904a94..b8a571303e1120689c4fee6af60592e9227c6ffb 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexDefinition.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexDefinition.java
@@ -84,7 +84,7 @@ public class UserIndexDefinition implements IndexDefinition {
     mapping.createDateTimeField(FIELD_CREATED_AT);
     mapping.createDateTimeField(FIELD_UPDATED_AT);
     mapping.createBooleanField(FIELD_ACTIVE);
-    mapping.stringFieldBuilder(FIELD_SCM_ACCOUNTS).build();
+    mapping.stringFieldBuilder(FIELD_SCM_ACCOUNTS).disableNorms().build();
   }
 
   private static SortedMap<String, String> buildGramSearchField() {

diff --git a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexDefinition.java
index 90dc408de33866f45cf25bc0f391eb77fe340613..59c0fa86d57ae6367db50c01c12793c98f98d355 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexDefinition.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexDefinition.java
@@ -51,7 +51,7 @@ public class ViewIndexDefinition implements IndexDefinition {
     // type "view"
     NewIndex.NewIndexType mapping = index.createType(TYPE_VIEW);
     mapping.setAttribute("_id", ImmutableMap.of("path", FIELD_UUID));
-    mapping.stringFieldBuilder(FIELD_UUID).build();
-    mapping.stringFieldBuilder(FIELD_PROJECTS).build();
+    mapping.stringFieldBuilder(FIELD_UUID).disableNorms().build();
+    mapping.stringFieldBuilder(FIELD_PROJECTS).disableNorms().build();
   }
 }