From d3e1ded1cc573404fa4f5b6e2032b8ed3e2fda4f Mon Sep 17 00:00:00 2001 From: Stephane Gamard Date: Wed, 28 May 2014 23:38:16 +0200 Subject: [PATCH] DAOv.2 - Dynamic gone with fixed mapping done with IndexField --- .../sonar/server/rule/index/RuleIndex.java | 107 +------------- .../server/rule/index/RuleNormalizer.java | 7 +- .../org/sonar/server/search/BaseIndex.java | 132 ++++++++++++++++-- .../org/sonar/server/search/IndexField.java | 24 ++-- .../org/sonar/server/search/Indexable.java | 9 ++ .../org/sonar/server/search/QueryOptions.java | 2 +- .../rule/index/RuleIndexMediumTest.java | 8 +- 7 files changed, 153 insertions(+), 136 deletions(-) diff --git a/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java b/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java index 31e4968bdf5..9b3f8699085 100644 --- a/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java +++ b/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java @@ -24,7 +24,6 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.BoolFilterBuilder; import org.elasticsearch.index.query.FilterBuilder; import org.elasticsearch.index.query.FilterBuilders; @@ -73,108 +72,6 @@ public class RuleIndex extends BaseIndex { .build(); } - - private void addFieldMapping(IndexField field, XContentBuilder mapping) throws IOException { - switch (field.type()) { - case STRING: - break; - case TEXT: - break; - case DATE: - break; - case NUMERIC: - break; - case BOOLEAN: - break; - case OBJECT: - break; - } - - - mapping.startObject(field.field()) - .field("type", "multi_field") - .startObject("fields") - .startObject(field.field()) - .field("type", "string") - .field("index", "analyzed") - .field("analyzer", "whitespace") - .endObject(); - - if (field.sortable()) { - mapping.startObject("sort") - .field("type", "string") - .field("index", "analyzed") - .field("analyzer", "sortable") - .endObject(); - } - - if (field.searchable()) { - mapping.startObject("search") - .field("type", "string") - .field("index", "analyzed") - .field("index_analyzer", "string_gram") - .field("search_analyzer", "standard") - .endObject(); - } - mapping.endObject() - .endObject(); - } - - private Map mapField(IndexField field) { - if(field.type() == IndexField.Type.TEXT){ - return mapTextField(field); - } else if(field.type() == IndexField.Type.STRING - || field.type() == IndexField.Type.KEY){ - return mapStringField(field); - } else if(field.type() == IndexField.Type.OBJECT){ - return mapObjectField(field); - } else if(field.type() == IndexField.Type.BOOLEAN){ - return mapBooleanField(field); - } else if(field.type() == IndexField.Type.DATE){ - return mapDateField(field); - } else { - throw new IllegalStateException("Mapping does not exist for type: " + field.type()); - } - } - - private Map mapBooleanField(IndexField field) { - Map mapping = new HashMap(); - mapping.put("type", "boolean"); - return mapping; - } - - private Map mapObjectField(IndexField field) { - Map mapping = new HashMap(); - mapping.put("type", "nested"); - mapping.put("index", "analyzed"); - mapping.put("dynamic", "true"); - return mapping; - } - - private Map mapStringField(IndexField field) { - Map mapping = new HashMap(); - mapping.put("type", "string"); - mapping.put("index", "analyzed"); - mapping.put("analyzer", "whitespace"); - return mapping; - } - - private Map mapDateField(IndexField field) { - Map mapping = new HashMap(); - mapping.put("type", "date"); - mapping.put("format", "date_time"); - return mapping; - } - - private Map mapTextField(IndexField field) { - Map mapping = new HashMap(); - mapping.put("type", "string"); - mapping.put("index", "analyzed"); - mapping.put("analyzer", "whitespace"); - return mapping; - } - - @Override protected Map mapKey() { Map mapping = new HashMap(); @@ -189,9 +86,13 @@ public class RuleIndex extends BaseIndex { for (IndexField field : RuleNormalizer.RuleField.ALL_FIELDS) { mapping.put(field.field(), mapField(field)); } + + System.out.println("mapping = " + mapping); return mapping; + + /* XContentBuilder mapping = jsonBuilder().startObject() .startObject(this.indexDefinition.getIndexType()) diff --git a/sonar-server/src/main/java/org/sonar/server/rule/index/RuleNormalizer.java b/sonar-server/src/main/java/org/sonar/server/rule/index/RuleNormalizer.java index efb9cbfc002..98e9a90e055 100644 --- a/sonar-server/src/main/java/org/sonar/server/rule/index/RuleNormalizer.java +++ b/sonar-server/src/main/java/org/sonar/server/rule/index/RuleNormalizer.java @@ -43,9 +43,10 @@ public class RuleNormalizer extends BaseNormalizer { public static class RuleField extends Indexable { - public static IndexField KEY = add(IndexField.Type.KEY, "key"); + public static IndexField KEY = addSearchable(IndexField.Type.STRING, "key"); public static IndexField REPOSITORY = add(IndexField.Type.STRING, "repo"); - public static IndexField NAME = addSortable(IndexField.Type.STRING, "name"); + public static IndexField NAME = addSortableAndSearchable(IndexField.Type.STRING, "name"); + public static IndexField CREATED_AT = addSortable(IndexField.Type.DATE, "createdAt"); public static IndexField UPDATED_AT = addSortable(IndexField.Type.DATE, "updatedAt"); public static IndexField HTML_DESCRIPTION = addSearchable(IndexField.Type.TEXT, "htmlDesc"); @@ -66,7 +67,7 @@ public class RuleNormalizer extends BaseNormalizer { public static IndexField NOTE_LOGIN = add(IndexField.Type.STRING, "noteLogin"); public static IndexField NOTE_CREATED_AT = add(IndexField.Type.DATE, "noteCreatedAt"); public static IndexField NOTE_UPDATED_AT = add(IndexField.Type.DATE, "noteUpdatedAt"); - public static IndexField _TAGS = addSearchable(IndexField.Type.TEXT, "_tags"); + public static IndexField _TAGS = addSearchable(IndexField.Type.STRING, "_tags"); } public static class RuleParamField extends Indexable { diff --git a/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java b/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java index be7ddaa8b3f..9033090623a 100644 --- a/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java +++ b/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java @@ -139,16 +139,14 @@ public abstract class BaseIndex, K extends Serializable> mapping.put("dynamic", false); mapping.put("_id", mapKey()); mapping.put("properties", mapProperties()); + LOG.debug("Index Mapping {}", mapping.get("properties")); - //mapping.put(this.getIndexType(), mapping); - - - LOG.info("Update of index {} for type {}", this.getIndexName(), this.getIndexType()); - getClient().admin().indices().preparePutMapping(index) - .setType(getIndexType()) - .setIgnoreConflicts(true) - .setSource(mapping) - .get(); + LOG.info("Update of index {} for type {}", this.getIndexName(), this.getIndexType()); + getClient().admin().indices().preparePutMapping(index) + .setType(getIndexType()) + .setIgnoreConflicts(true) + .setSource(mapping) + .get(); } catch (Exception e) { throw new IllegalStateException("Invalid configuration for index " + this.getIndexName(), e); @@ -204,6 +202,122 @@ public abstract class BaseIndex, K extends Serializable> protected abstract Map mapKey(); + + protected Map mapField(IndexField field) { + return mapField(field, true); + } + + protected Map mapField(IndexField field, boolean allowRecursive) { + if (field.type() == IndexField.Type.TEXT) { + return mapTextField(field, allowRecursive); + } else if (field.type() == IndexField.Type.STRING) { + return mapStringField(field, allowRecursive); + } else if (field.type() == IndexField.Type.OBJECT) { + return mapObjectField(field); + } else if (field.type() == IndexField.Type.BOOLEAN) { + return mapBooleanField(field); + } else if (field.type() == IndexField.Type.DATE) { + return mapDateField(field); + } else { + throw new IllegalStateException("Mapping does not exist for type: " + field.type()); + } + } + + protected Map mapBooleanField(IndexField field) { + Map mapping = new HashMap(); + mapping.put("type", "boolean"); + return mapping; + } + + protected Map mapObjectField(IndexField field) { + Map mapping = new HashMap(); + mapping.put("type", "nested"); + mapping.put("index", "analyzed"); + mapping.put("dynamic", "true"); + return mapping; + } + + protected Map mapDateField(IndexField field) { + Map mapping = new HashMap(); + mapping.put("type", "date"); + mapping.put("format", "date_time"); + return mapping; + } + + protected boolean needMultiField(IndexField field) { + return ((field.type() == IndexField.Type.TEXT + || field.type() == IndexField.Type.STRING) + && (field.sortable() || field.searchable())); + } + + protected Map mapSortField(IndexField field) { + Map mapping = new HashMap(); + mapping.put("type", "string"); + mapping.put("index", "analyzed"); + mapping.put("analyzer", "sortable"); + return mapping; + } + + protected Map mapGramsField(IndexField field) { + Map mapping = new HashMap(); + mapping.put("type", "string"); + mapping.put("index", "analyzed"); + mapping.put("index_analyzer", "index_grams"); + mapping.put("search_analyzer", "search_grams"); + return mapping; + } + + protected Map mapWordsField(IndexField field) { + Map mapping = new HashMap(); + mapping.put("type", "string"); + mapping.put("index", "analyzed"); + mapping.put("index_analyzer", "index_words"); + mapping.put("search_analyzer", "search_words"); + return mapping; + } + + protected Map mapMultiField(IndexField field) { + Map mapping = new HashMap(); + if (field.sortable()) { + mapping.put(IndexField.SORT_SUFFIX, mapSortField(field)); + } + if (field.searchable()) { + if (field.type() != IndexField.Type.TEXT) { + mapping.put(IndexField.SEARCH_PARTIAL_SUFFIX, mapGramsField(field)); + } + mapping.put(IndexField.SEARCH_WORDS_SUFFIX, mapWordsField(field)); + } + mapping.put(field.field(), mapField(field, false)); + return mapping; + } + + protected Map mapStringField(IndexField field, boolean allowRecursive) { + Map mapping = new HashMap(); + // check if the field needs to be MultiField + if (allowRecursive && needMultiField(field)) { + mapping.put("type", "multi_field"); + mapping.put("fields", mapMultiField(field)); + } else { + mapping.put("type", "string"); + mapping.put("index", "analyzed"); + mapping.put("analyzer", "keyword"); + } + return mapping; + } + + protected Map mapTextField(IndexField field, boolean allowRecursive) { + Map mapping = new HashMap(); + // check if the field needs to be MultiField + if (allowRecursive && needMultiField(field)) { + mapping.put("type", "multi_field"); + mapping.put("fields", mapMultiField(field)); + } else { + mapping.put("type", "string"); + mapping.put("index", "not_analyzed"); + } + return mapping; + } + @Override public void refresh() { getClient() diff --git a/sonar-server/src/main/java/org/sonar/server/search/IndexField.java b/sonar-server/src/main/java/org/sonar/server/search/IndexField.java index 231b031cf80..630d66ed450 100644 --- a/sonar-server/src/main/java/org/sonar/server/search/IndexField.java +++ b/sonar-server/src/main/java/org/sonar/server/search/IndexField.java @@ -25,22 +25,25 @@ import org.apache.commons.lang.builder.ToStringStyle; public class IndexField { public static enum Type { - KEY, STRING, TEXT, DATE, BOOLEAN, NUMERIC, OBJECT + STRING, TEXT, DATE, BOOLEAN, NUMERIC, OBJECT } + public static final String SORT_SUFFIX = "sort"; + public static final String SEARCH_WORDS_SUFFIX = "words"; + public static final String SEARCH_PARTIAL_SUFFIX = "grams"; + private final Type type; private final String field; - private boolean sortable; - private boolean searchable; - private boolean matchable; + private boolean sortable = false; + private boolean searchable = false; IndexField(Type type, String field) { this.type = type; this.field = field; } - public Boolean sortable() { + public boolean sortable() { return sortable; } @@ -49,7 +52,7 @@ public class IndexField { return this; } - public Boolean searchable() { + public boolean searchable() { return searchable; } @@ -58,15 +61,6 @@ public class IndexField { return this; } - public Boolean matchable() { - return matchable; - } - - public IndexField matchable(Boolean matchable) { - this.matchable = matchable; - return this; - } - public Type type() { return type; } diff --git a/sonar-server/src/main/java/org/sonar/server/search/Indexable.java b/sonar-server/src/main/java/org/sonar/server/search/Indexable.java index 97f24ba60ea..57037069429 100644 --- a/sonar-server/src/main/java/org/sonar/server/search/Indexable.java +++ b/sonar-server/src/main/java/org/sonar/server/search/Indexable.java @@ -46,6 +46,15 @@ public class Indexable { return indexField; } + + public static IndexField addSortableAndSearchable(IndexField.Type type, String field) { + IndexField indexField = new IndexField(type, field) + .searchable(true) + .sortable(true); + ALL_FIELDS.add(indexField); + return indexField; + } + public static IndexField addSortable(IndexField.Type type, String field){ IndexField indexField = new IndexField(type, field) .sortable(true); diff --git a/sonar-server/src/main/java/org/sonar/server/search/QueryOptions.java b/sonar-server/src/main/java/org/sonar/server/search/QueryOptions.java index 5184d6323b4..881b4d87470 100644 --- a/sonar-server/src/main/java/org/sonar/server/search/QueryOptions.java +++ b/sonar-server/src/main/java/org/sonar/server/search/QueryOptions.java @@ -34,7 +34,7 @@ import java.util.Set; */ public class QueryOptions { - public static final QueryOptions DEFAULT = new QueryOptions(); + public static final QueryOptions DEFAULT = new QueryOptions().setLimit(Integer.MAX_VALUE); public static final int DEFAULT_OFFSET = 0; public static final int DEFAULT_LIMIT = 10; diff --git a/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexMediumTest.java b/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexMediumTest.java index e091a79a981..51b32118ecf 100644 --- a/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexMediumTest.java +++ b/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexMediumTest.java @@ -234,8 +234,6 @@ public class RuleIndexMediumTest { dao.insert(dbSession, newRuleDto(RuleKey.of("java", "S002"))); dbSession.commit(); - - Thread.sleep(1000000); Result results = index.search(new RuleQuery(), new QueryOptions()); assertThat(results.getTotal()).isEqualTo(2); @@ -337,15 +335,15 @@ public class RuleIndexMediumTest { query = new RuleQuery().setDebtCharacteristics(ImmutableSet.of(char11.getKey(), char1.getKey())); assertThat(index.search(query, new QueryOptions()).getHits()).hasSize(1); - // search by Char + // match by Char query = new RuleQuery().setQueryText(char1.getKey()); assertThat(index.search(query, new QueryOptions()).getHits()).hasSize(1); - // search by SubChar + // match by SubChar query = new RuleQuery().setQueryText(char11.getKey()); assertThat(index.search(query, new QueryOptions()).getHits()).hasSize(1); - // search by SubChar & Char + // match by SubChar & Char query = new RuleQuery().setQueryText(char11.getKey() + " " + char1.getKey()); assertThat(index.search(query, new QueryOptions()).getHits()).hasSize(1); } -- 2.39.5