DAOv.2 - Dynamic gone with fixed mapping done with IndexField

diff --git a/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java b/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java
index 31e4968bdf53e25af7e77ea30649c05820309d6c..9b3f86990852925f6995d3c5a82fc35be7b2dcd3 100644 (file)
--- a/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java
+++ b/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.BoolFilterBuilder;
 import org.elasticsearch.index.query.FilterBuilder;
 import org.elasticsearch.index.query.FilterBuilders;
@@ -73,108 +72,6 @@ public class RuleIndex extends BaseIndex<Rule, RuleDto, RuleKey> {
       .build();
   }
 
-
-  private void addFieldMapping(IndexField field, XContentBuilder mapping) throws IOException {
-    switch (field.type()) {
-      case STRING:
-        break;
-      case TEXT:
-        break;
-      case DATE:
-        break;
-      case NUMERIC:
-        break;
-      case BOOLEAN:
-        break;
-      case OBJECT:
-        break;
-    }
-
-
-    mapping.startObject(field.field())
-      .field("type", "multi_field")
-      .startObject("fields")
-      .startObject(field.field())
-      .field("type", "string")
-      .field("index", "analyzed")
-      .field("analyzer", "whitespace")
-      .endObject();
-
-    if (field.sortable()) {
-      mapping.startObject("sort")
-        .field("type", "string")
-        .field("index", "analyzed")
-        .field("analyzer", "sortable")
-        .endObject();
-    }
-
-    if (field.searchable()) {
-      mapping.startObject("search")
-        .field("type", "string")
-        .field("index", "analyzed")
-        .field("index_analyzer", "string_gram")
-        .field("search_analyzer", "standard")
-        .endObject();
-    }
-    mapping.endObject()
-      .endObject();
-  }
-
-  private Map mapField(IndexField field) {
-    if(field.type() == IndexField.Type.TEXT){
-      return mapTextField(field);
-    } else if(field.type() == IndexField.Type.STRING
-      || field.type() == IndexField.Type.KEY){
-      return mapStringField(field);
-    } else if(field.type() == IndexField.Type.OBJECT){
-      return mapObjectField(field);
-    } else if(field.type() == IndexField.Type.BOOLEAN){
-      return mapBooleanField(field);
-    } else if(field.type() == IndexField.Type.DATE){
-      return mapDateField(field);
-    } else {
-      throw new IllegalStateException("Mapping does not exist for type: " + field.type());
-    }
-  }
-
-  private Map mapBooleanField(IndexField field) {
-    Map<String, Object> mapping = new HashMap<String, Object>();
-    mapping.put("type", "boolean");
-    return mapping;
-  }
-
-  private Map mapObjectField(IndexField field) {
-    Map<String, Object> mapping = new HashMap<String, Object>();
-    mapping.put("type", "nested");
-    mapping.put("index", "analyzed");
-    mapping.put("dynamic", "true");
-    return mapping;
-  }
-
-  private Map mapStringField(IndexField field) {
-    Map<String, Object> mapping = new HashMap<String, Object>();
-    mapping.put("type", "string");
-    mapping.put("index", "analyzed");
-    mapping.put("analyzer", "whitespace");
-    return mapping;
-  }
-
-  private Map mapDateField(IndexField field) {
-    Map<String, Object> mapping = new HashMap<String, Object>();
-    mapping.put("type", "date");
-    mapping.put("format", "date_time");
-    return mapping;
-  }
-
-  private Map mapTextField(IndexField field) {
-    Map<String, Object> mapping = new HashMap<String, Object>();
-    mapping.put("type", "string");
-    mapping.put("index", "analyzed");
-    mapping.put("analyzer", "whitespace");
-    return mapping;
-  }
-
-
   @Override
   protected Map mapKey() {
     Map<String, Object> mapping = new HashMap<String, Object>();
@@ -189,9 +86,13 @@ public class RuleIndex extends BaseIndex<Rule, RuleDto, RuleKey> {
     for (IndexField field : RuleNormalizer.RuleField.ALL_FIELDS) {
       mapping.put(field.field(), mapField(field));
     }
+
+    System.out.println("mapping = " + mapping);
     return mapping;
 
 
+
+
     /*
     XContentBuilder mapping = jsonBuilder().startObject()
       .startObject(this.indexDefinition.getIndexType())

diff --git a/sonar-server/src/main/java/org/sonar/server/rule/index/RuleNormalizer.java b/sonar-server/src/main/java/org/sonar/server/rule/index/RuleNormalizer.java
index efb9cbfc002c4055aa24003feb1ebd587c7a197a..98e9a90e0556e74e1eff602f5f5983ce550e5300 100644 (file)
--- a/sonar-server/src/main/java/org/sonar/server/rule/index/RuleNormalizer.java
+++ b/sonar-server/src/main/java/org/sonar/server/rule/index/RuleNormalizer.java
@@ -43,9 +43,10 @@ public class RuleNormalizer extends BaseNormalizer<RuleDto, RuleKey> {
 
   public static class RuleField extends Indexable {
 
-    public static IndexField KEY = add(IndexField.Type.KEY, "key");
+    public static IndexField KEY = addSearchable(IndexField.Type.STRING, "key");
     public static IndexField REPOSITORY = add(IndexField.Type.STRING, "repo");
-    public static IndexField NAME = addSortable(IndexField.Type.STRING, "name");
+    public static IndexField NAME = addSortableAndSearchable(IndexField.Type.STRING, "name");
+
     public static IndexField CREATED_AT = addSortable(IndexField.Type.DATE, "createdAt");
     public static IndexField UPDATED_AT = addSortable(IndexField.Type.DATE, "updatedAt");
     public static IndexField HTML_DESCRIPTION = addSearchable(IndexField.Type.TEXT, "htmlDesc");
@@ -66,7 +67,7 @@ public class RuleNormalizer extends BaseNormalizer<RuleDto, RuleKey> {
     public static IndexField NOTE_LOGIN = add(IndexField.Type.STRING, "noteLogin");
     public static IndexField NOTE_CREATED_AT = add(IndexField.Type.DATE, "noteCreatedAt");
     public static IndexField NOTE_UPDATED_AT = add(IndexField.Type.DATE, "noteUpdatedAt");
-    public static IndexField _TAGS = addSearchable(IndexField.Type.TEXT, "_tags");
+    public static IndexField _TAGS = addSearchable(IndexField.Type.STRING, "_tags");
   }
 
   public static class RuleParamField extends Indexable {

diff --git a/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java b/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java
index be7ddaa8b3fb9bd7ef7645e167449d0a13fcc77a..9033090623a78a14a47a7fe4ac7197614c4f414f 100644 (file)
--- a/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java
+++ b/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java
@@ -139,16 +139,14 @@ public abstract class BaseIndex<D, E extends Dto<K>, K extends Serializable>
       mapping.put("dynamic", false);
       mapping.put("_id", mapKey());
       mapping.put("properties", mapProperties());
+      LOG.debug("Index Mapping {}", mapping.get("properties"));
 
-      //mapping.put(this.getIndexType(), mapping);
-
-
-        LOG.info("Update of index {} for type {}", this.getIndexName(), this.getIndexType());
-        getClient().admin().indices().preparePutMapping(index)
-          .setType(getIndexType())
-          .setIgnoreConflicts(true)
-          .setSource(mapping)
-          .get();
+      LOG.info("Update of index {} for type {}", this.getIndexName(), this.getIndexType());
+      getClient().admin().indices().preparePutMapping(index)
+        .setType(getIndexType())
+        .setIgnoreConflicts(true)
+        .setSource(mapping)
+        .get();
 
     } catch (Exception e) {
       throw new IllegalStateException("Invalid configuration for index " + this.getIndexName(), e);
@@ -204,6 +202,122 @@ public abstract class BaseIndex<D, E extends Dto<K>, K extends Serializable>
 
   protected abstract Map mapKey();
 
+
+  protected Map mapField(IndexField field) {
+    return mapField(field, true);
+  }
+
+  protected Map mapField(IndexField field, boolean allowRecursive) {
+    if (field.type() == IndexField.Type.TEXT) {
+      return mapTextField(field, allowRecursive);
+    } else if (field.type() == IndexField.Type.STRING) {
+      return mapStringField(field, allowRecursive);
+    } else if (field.type() == IndexField.Type.OBJECT) {
+      return mapObjectField(field);
+    } else if (field.type() == IndexField.Type.BOOLEAN) {
+      return mapBooleanField(field);
+    } else if (field.type() == IndexField.Type.DATE) {
+      return mapDateField(field);
+    } else {
+      throw new IllegalStateException("Mapping does not exist for type: " + field.type());
+    }
+  }
+
+  protected Map mapBooleanField(IndexField field) {
+    Map<String, Object> mapping = new HashMap<String, Object>();
+    mapping.put("type", "boolean");
+    return mapping;
+  }
+
+  protected Map mapObjectField(IndexField field) {
+    Map<String, Object> mapping = new HashMap<String, Object>();
+    mapping.put("type", "nested");
+    mapping.put("index", "analyzed");
+    mapping.put("dynamic", "true");
+    return mapping;
+  }
+
+  protected Map mapDateField(IndexField field) {
+    Map<String, Object> mapping = new HashMap<String, Object>();
+    mapping.put("type", "date");
+    mapping.put("format", "date_time");
+    return mapping;
+  }
+
+  protected boolean needMultiField(IndexField field) {
+    return ((field.type() == IndexField.Type.TEXT
+      || field.type() == IndexField.Type.STRING)
+      && (field.sortable() || field.searchable()));
+  }
+
+  protected Map mapSortField(IndexField field) {
+    Map<String, Object> mapping = new HashMap<String, Object>();
+    mapping.put("type", "string");
+    mapping.put("index", "analyzed");
+    mapping.put("analyzer", "sortable");
+    return mapping;
+  }
+
+  protected Map mapGramsField(IndexField field) {
+    Map<String, Object> mapping = new HashMap<String, Object>();
+    mapping.put("type", "string");
+    mapping.put("index", "analyzed");
+    mapping.put("index_analyzer", "index_grams");
+    mapping.put("search_analyzer", "search_grams");
+    return mapping;
+  }
+
+  protected Map mapWordsField(IndexField field) {
+    Map<String, Object> mapping = new HashMap<String, Object>();
+    mapping.put("type", "string");
+    mapping.put("index", "analyzed");
+    mapping.put("index_analyzer", "index_words");
+    mapping.put("search_analyzer", "search_words");
+    return mapping;
+  }
+
+  protected Map mapMultiField(IndexField field) {
+    Map<String, Object> mapping = new HashMap<String, Object>();
+    if (field.sortable()) {
+      mapping.put(IndexField.SORT_SUFFIX, mapSortField(field));
+    }
+    if (field.searchable()) {
+      if (field.type() != IndexField.Type.TEXT) {
+        mapping.put(IndexField.SEARCH_PARTIAL_SUFFIX, mapGramsField(field));
+      }
+      mapping.put(IndexField.SEARCH_WORDS_SUFFIX, mapWordsField(field));
+    }
+    mapping.put(field.field(), mapField(field, false));
+    return mapping;
+  }
+
+  protected Map mapStringField(IndexField field, boolean allowRecursive) {
+    Map<String, Object> mapping = new HashMap<String, Object>();
+    // check if the field needs to be MultiField
+    if (allowRecursive && needMultiField(field)) {
+      mapping.put("type", "multi_field");
+      mapping.put("fields", mapMultiField(field));
+    } else {
+      mapping.put("type", "string");
+      mapping.put("index", "analyzed");
+      mapping.put("analyzer", "keyword");
+    }
+    return mapping;
+  }
+
+  protected Map mapTextField(IndexField field, boolean allowRecursive) {
+    Map<String, Object> mapping = new HashMap<String, Object>();
+    // check if the field needs to be MultiField
+    if (allowRecursive && needMultiField(field)) {
+      mapping.put("type", "multi_field");
+      mapping.put("fields", mapMultiField(field));
+    } else {
+      mapping.put("type", "string");
+      mapping.put("index", "not_analyzed");
+    }
+    return mapping;
+  }
+
   @Override
   public void refresh() {
     getClient()

diff --git a/sonar-server/src/main/java/org/sonar/server/search/IndexField.java b/sonar-server/src/main/java/org/sonar/server/search/IndexField.java
index 231b031cf802b6705f8f360affecb1efbe7d5983..630d66ed4502f88d7bfe265bcb1b5d47cc288c29 100644 (file)
--- a/sonar-server/src/main/java/org/sonar/server/search/IndexField.java
+++ b/sonar-server/src/main/java/org/sonar/server/search/IndexField.java
@@ -25,22 +25,25 @@ import org.apache.commons.lang.builder.ToStringStyle;
 public class IndexField {
 
   public static enum Type {
-    KEY, STRING, TEXT, DATE, BOOLEAN, NUMERIC, OBJECT
+    STRING, TEXT, DATE, BOOLEAN, NUMERIC, OBJECT
   }
 
+  public static final String SORT_SUFFIX = "sort";
+  public static final String SEARCH_WORDS_SUFFIX = "words";
+  public static final String SEARCH_PARTIAL_SUFFIX = "grams";
+
   private final Type type;
   private final String field;
 
-  private boolean sortable;
-  private boolean searchable;
-  private boolean matchable;
+  private boolean sortable = false;
+  private boolean searchable = false;
 
   IndexField(Type type, String field) {
     this.type = type;
     this.field = field;
   }
 
-  public Boolean sortable() {
+  public boolean sortable() {
     return sortable;
   }
 
@@ -49,7 +52,7 @@ public class IndexField {
     return this;
   }
 
-  public Boolean searchable() {
+  public boolean searchable() {
     return searchable;
   }
 
@@ -58,15 +61,6 @@ public class IndexField {
     return this;
   }
 
-  public Boolean matchable() {
-    return matchable;
-  }
-
-  public IndexField matchable(Boolean matchable) {
-    this.matchable = matchable;
-    return this;
-  }
-
   public Type type() {
     return type;
   }

diff --git a/sonar-server/src/main/java/org/sonar/server/search/Indexable.java b/sonar-server/src/main/java/org/sonar/server/search/Indexable.java
index 97f24ba60eac577c753e39eac6a54b460e32abf6..570370694299e4bedb5fbd1727d68b34d3399342 100644 (file)
--- a/sonar-server/src/main/java/org/sonar/server/search/Indexable.java
+++ b/sonar-server/src/main/java/org/sonar/server/search/Indexable.java
@@ -46,6 +46,15 @@ public class Indexable {
     return indexField;
   }
 
+
+  public static IndexField addSortableAndSearchable(IndexField.Type type, String field) {
+    IndexField indexField = new IndexField(type, field)
+      .searchable(true)
+      .sortable(true);
+    ALL_FIELDS.add(indexField);
+    return indexField;
+  }
+
   public static IndexField addSortable(IndexField.Type type, String field){
     IndexField indexField = new IndexField(type, field)
       .sortable(true);

diff --git a/sonar-server/src/main/java/org/sonar/server/search/QueryOptions.java b/sonar-server/src/main/java/org/sonar/server/search/QueryOptions.java
index 5184d6323b48db1f13d44e1d8e2f9bc66aa1847c..881b4d874704b92d7e9f3bacdb543376d7a4882a 100644 (file)
--- a/sonar-server/src/main/java/org/sonar/server/search/QueryOptions.java
+++ b/sonar-server/src/main/java/org/sonar/server/search/QueryOptions.java
@@ -34,7 +34,7 @@ import java.util.Set;
  */
 public class QueryOptions {
 
-  public static final QueryOptions DEFAULT = new QueryOptions();
+  public static final QueryOptions DEFAULT = new QueryOptions().setLimit(Integer.MAX_VALUE);
 
   public static final int DEFAULT_OFFSET = 0;
   public static final int DEFAULT_LIMIT = 10;

diff --git a/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexMediumTest.java b/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexMediumTest.java
index e091a79a981cb81b15641d3596517d2f9ed3a826..51b32118ecfeddb71ba2c495b0bbf535c28d6059 100644 (file)
--- a/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexMediumTest.java
+++ b/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexMediumTest.java
@@ -234,8 +234,6 @@ public class RuleIndexMediumTest {
     dao.insert(dbSession, newRuleDto(RuleKey.of("java", "S002")));
     dbSession.commit();
 
-
-    Thread.sleep(1000000);
     Result results = index.search(new RuleQuery(), new QueryOptions());
 
     assertThat(results.getTotal()).isEqualTo(2);
@@ -337,15 +335,15 @@ public class RuleIndexMediumTest {
     query = new RuleQuery().setDebtCharacteristics(ImmutableSet.of(char11.getKey(), char1.getKey()));
     assertThat(index.search(query, new QueryOptions()).getHits()).hasSize(1);
 
-    // search by Char
+    // match by Char
     query = new RuleQuery().setQueryText(char1.getKey());
     assertThat(index.search(query, new QueryOptions()).getHits()).hasSize(1);
 
-    // search by SubChar
+    // match by SubChar
     query = new RuleQuery().setQueryText(char11.getKey());
     assertThat(index.search(query, new QueryOptions()).getHits()).hasSize(1);
 
-    // search by SubChar & Char
+    // match by SubChar & Char
     query = new RuleQuery().setQueryText(char11.getKey() + " " + char1.getKey());
     assertThat(index.search(query, new QueryOptions()).getHits()).hasSize(1);
   }