From 9167df88f5f49abf5707264b80008f3813ca05d8 Mon Sep 17 00:00:00 2001
From: Stephane Gamard <stephane.gamard@searchbox.com>
Date: Wed, 28 May 2014 10:47:22 +0200
Subject: [PATCH] DAOv.2 - Updated Core Analyzer for ES (required for sort,
 match and search in CodingRules)

---
 .../java/org/sonar/server/search/ESNode.java  | 16 +++++-----
 .../org/sonar/server/search/ESNodeTest.java   | 30 +++++++++++++++----
 2 files changed, 33 insertions(+), 13 deletions(-)

diff --git a/sonar-server/src/main/java/org/sonar/server/search/ESNode.java b/sonar-server/src/main/java/org/sonar/server/search/ESNode.java
index 0f69106a5fb..b82aba120f7 100644
--- a/sonar-server/src/main/java/org/sonar/server/search/ESNode.java
+++ b/sonar-server/src/main/java/org/sonar/server/search/ESNode.java
@@ -130,16 +130,16 @@ public class ESNode implements Startable {
 
       .put("index.analysis.analyzer.sortable.type", "custom")
       .put("index.analysis.analyzer.sortable.tokenizer", "keyword")
-      .put("index.analysis.analyzer.sortable.filter", "lowercase")
+      .putArray("index.analysis.analyzer.sortable.filter", "trim","lowercase","truncate")
 
-      .put("index.analysis.analyzer.rule_name.type", "custom")
-      .put("index.analysis.analyzer.rule_name.tokenizer", "standard")
-      .putArray("index.analysis.analyzer.rule_name.filter", "lowercase", "rule_name_ngram")
+      .put("index.analysis.analyzer.string_gram.type", "custom")
+      .put("index.analysis.analyzer.string_gram.tokenizer", "whitespace")
+      .putArray("index.analysis.analyzer.string_gram.filter", "lowercase", "code_gram")
 
-      .put("index.analysis.filter.rule_name_ngram.type", "nGram")
-      .put("index.analysis.filter.rule_name_ngram.min_gram", 3)
-      .put("index.analysis.filter.rule_name_ngram.max_gram", 5)
-      .putArray("index.analysis.filter.rule_name_ngram.token_chars", "letter", "digit");
+      .put("index.analysis.filter.code_gram.type", "edgeNGram")
+      .put("index.analysis.filter.code_gram.min_gram", 2)
+      .put("index.analysis.filter.code_gram.max_gram", 15)
+      .putArray("index.analysis.filter.code_gram.token_chars", "letter", "digit", "punctuation", "symbol");
 
     initDirs(esSettings);
     initRestConsole(esSettings);
diff --git a/sonar-server/src/test/java/org/sonar/server/search/ESNodeTest.java b/sonar-server/src/test/java/org/sonar/server/search/ESNodeTest.java
index 5f9b5a8ec48..5f990ed4faa 100644
--- a/sonar-server/src/test/java/org/sonar/server/search/ESNodeTest.java
+++ b/sonar-server/src/test/java/org/sonar/server/search/ESNodeTest.java
@@ -22,6 +22,7 @@ package org.sonar.server.search;
 import com.google.common.io.Resources;
 import org.apache.commons.io.FileUtils;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
 import org.elasticsearch.client.AdminClient;
 import org.elasticsearch.client.ClusterAdminClient;
 import org.elasticsearch.cluster.ClusterState;
@@ -102,11 +103,6 @@ public class ESNodeTest {
       .execute().actionGet();
     node.client().admin().cluster().prepareHealth("polop").setWaitForYellowStatus().get(TimeValue.timeValueMillis(1000));
 
-    // default "sortable" analyzer is defined for all indices
-    assertThat(node.client().admin().indices()
-      .prepareAnalyze("polop", "This Is A Wonderful Text").setAnalyzer("sortable").get()
-      .getTokens().get(0).getTerm()).isEqualTo("this is a wonderful text");
-
     // strict mapping is enforced
     try {
       node.client().prepareIndex("polop", "type1", "666").setSource(
@@ -117,6 +113,30 @@ public class ESNodeTest {
     }
   }
 
+  @Test
+  public void sortable_analyzer() throws Exception {
+    ESNode node = new ESNode(fs, new Settings());
+    node.start();
+
+    node.client().admin().indices().prepareCreate("polop")
+      .addMapping("type1", "{\"type1\": {\"properties\": {\"value\": {\"type\": \"string\"}}}}")
+      .execute().actionGet();
+    node.client().admin().cluster().prepareHealth("polop").setWaitForYellowStatus().get(TimeValue.timeValueMillis(1000));
+
+    // default "sortable" analyzer is defined for all indices
+    assertThat(node.client().admin().indices()
+      .prepareAnalyze("polop", "This Is A Wonderful Text").setAnalyzer("sortable").get()
+      .getTokens().get(0).getTerm()).isEqualTo("this is a ");
+
+    // default "sortable" analyzer is defined for all indices
+    AnalyzeResponse response = node.client().admin().indices()
+      .prepareAnalyze("polop", "he.llo w@rl#d").setAnalyzer("string_gram").get();
+
+    assertThat(response.getTokens()).hasSize(10);
+    assertThat(response.getTokens().get(0).getTerm()).isEqualTo("he");
+    assertThat(response.getTokens().get(7).getTerm()).isEqualTo("w@rl");
+  }
+
   @Test
   public void should_restore_status_on_startup() throws Exception {
     File zip = new File(Resources.getResource(getClass(), "ESNodeTest/data-es-clean.zip").toURI());
-- 
2.39.5