From: Simon Brandhof Date: Wed, 29 Oct 2014 08:24:27 +0000 (+0100) Subject: Refactor ES framework X-Git-Tag: 5.0-RC1~517 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=30c42f7f010ce7e63f3c357130f485de0284b0f8;p=sonarqube.git Refactor ES framework * do not limit facet values to integers, but use long * distinguish integer, long and double field types * remove unused sub-facets --- diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/InternalRubyIssueService.java b/server/sonar-server/src/main/java/org/sonar/server/issue/InternalRubyIssueService.java index 1513c8b2936..a686986824b 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/InternalRubyIssueService.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/InternalRubyIssueService.java @@ -111,7 +111,7 @@ public class InternalRubyIssueService implements ServerComponent { return issueService.getByKey(issueKey); } - public Map findIssueAssignees(Map params) { + public Map findIssueAssignees(Map params) { return issueService.findIssueAssignees(issueQueryService.createFromMap(params)); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/IssueService.java b/server/sonar-server/src/main/java/org/sonar/server/issue/IssueService.java index e2100f8a6ad..efa6867f785 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/IssueService.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/IssueService.java @@ -292,8 +292,8 @@ public class IssueService implements ServerComponent { return aggregation; } - public Map findIssueAssignees(IssueQuery query) { - Map result = newLinkedHashMap(); + public Map findIssueAssignees(IssueQuery query) { + Map result = newLinkedHashMap(); List facetValues = indexClient.get(IssueIndex.class).listAssignees(query); for (FacetValue facetValue : facetValues) { if ("_notAssigned_".equals(facetValue.getKey())) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java index 86ef7d40b5c..0f96719f3b8 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java @@ -165,9 +165,9 @@ public class IssueIndex extends BaseIndex { Terms aggregation = (Terms) response.getAggregations().getAsMap().get(IssueNormalizer.IssueField.ASSIGNEE.field()); List facetValues = newArrayList(); for (Terms.Bucket value : aggregation.getBuckets()) { - facetValues.add(new FacetValue(value.getKey(), (int) value.getDocCount())); + facetValues.add(new FacetValue(value.getKey(), value.getDocCount())); } - facetValues.add(new FacetValue("_notAssigned_", (int) ((InternalMissing) response.getAggregations().get("notAssigned")).getDocCount())); + facetValues.add(new FacetValue("_notAssigned_", ((InternalMissing) response.getAggregations().get("notAssigned")).getDocCount())); return facetValues; } @@ -390,19 +390,18 @@ public class IssueIndex extends BaseIndex { } private void setSorting(IssueQuery query, SearchRequestBuilder esSearch) { - /* integrate Query Sort */ String sortField = query.sort(); - Boolean asc = query.asc(); if (sortField != null) { + Boolean asc = query.asc(); List fields = toIndexFields(sortField); for (IndexField field : fields) { - FieldSortBuilder sort = SortBuilders.fieldSort(field.sortField()); + FieldSortBuilder sortBuilder = SortBuilders.fieldSort(field.sortField()); if (asc != null && asc) { - sort.order(SortOrder.ASC); + sortBuilder.order(SortOrder.ASC); } else { - sort.order(SortOrder.DESC); + sortBuilder.order(SortOrder.DESC); } - esSearch.addSort(sort); + esSearch.addSort(sortBuilder); } } else { esSearch.addSort(IssueNormalizer.IssueField.ISSUE_UPDATED_AT.sortField(), SortOrder.DESC); diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueNormalizer.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueNormalizer.java index 94407f6de54..777ba466f28 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueNormalizer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueNormalizer.java @@ -58,18 +58,18 @@ public class IssueNormalizer extends BaseNormalizer { public static final IndexField ASSIGNEE = addSortable(IndexField.Type.STRING, "assignee"); public static final IndexField ATTRIBUTES = add(IndexField.Type.STRING, "attributes"); public static final IndexField AUTHOR_LOGIN = add(IndexField.Type.STRING, "authorLogin"); - public static final IndexField DEBT = add(IndexField.Type.NUMERIC, "debt"); - public static final IndexField EFFORT = add(IndexField.Type.NUMERIC, "effort"); + public static final IndexField DEBT = add(IndexField.Type.DOUBLE, "debt"); + public static final IndexField EFFORT = add(IndexField.Type.DOUBLE, "effort"); public static final IndexField ISSUE_CREATED_AT = addSortable(IndexField.Type.DATE, "issueCreatedAt"); public static final IndexField ISSUE_UPDATED_AT = addSortable(IndexField.Type.DATE, "issueUpdatedAt"); public static final IndexField ISSUE_CLOSE_DATE = addSortable(IndexField.Type.DATE, "issueClosedAt"); - public static final IndexField LINE = addSortable(IndexField.Type.NUMERIC, "line"); + public static final IndexField LINE = addSortable(IndexField.Type.INTEGER, "line"); public static final IndexField MESSAGE = add(IndexField.Type.STRING, "message"); public static final IndexField RESOLUTION = add(IndexField.Type.STRING, "resolution"); public static final IndexField REPORTER = add(IndexField.Type.STRING, "reporter"); public static final IndexField STATUS = addSortable(IndexField.Type.STRING, "status"); public static final IndexField SEVERITY = add(IndexField.Type.STRING, "severity"); - public static final IndexField SEVERITY_VALUE = addSortable(IndexField.Type.NUMERIC, "severityValue"); + public static final IndexField SEVERITY_VALUE = addSortable(IndexField.Type.INTEGER, "severityValue"); public static final IndexField LANGUAGE = add(IndexField.Type.STRING, "language"); public static final IndexField RULE_KEY = add(IndexField.Type.STRING, "ruleKey"); public static final IndexField FILE_PATH = addSortable(IndexField.Type.STRING, "filePath"); diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleNormalizer.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleNormalizer.java index 086a75fda73..54eae4488f7 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleNormalizer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleNormalizer.java @@ -78,7 +78,7 @@ public class RuleNormalizer extends BaseNormalizer { * SQALE console. */ @Deprecated - public static final IndexField ID = addSortable(IndexField.Type.NUMERIC, "id"); + public static final IndexField ID = addSortable(IndexField.Type.DOUBLE, "id"); public static final IndexField KEY = addSortable(IndexField.Type.STRING, "key"); public static final IndexField _KEY = add(IndexField.Type.STRING, "_key"); diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleQuery.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleQuery.java index ca8a14e6fe3..d7edab19ed6 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleQuery.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleQuery.java @@ -230,8 +230,8 @@ public class RuleQuery { } public RuleQuery setSortField(@Nullable IndexField sf) { - if (sf != null && !sf.sortable()) { - throw new IllegalStateException("Field '" + sf.field() + "' is not sortable!"); + if (sf != null && !sf.isSortable()) { + throw new IllegalStateException(String.format("Field '%s' is not sortable", sf.field())); } this.sortField = sf; return this; diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java b/server/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java index 2cf2f6af745..a4dae1a5819 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java @@ -268,8 +268,12 @@ public abstract class BaseIndex, KEY extends Serial return mapNestedField(field); } else if (field.type() == IndexField.Type.DATE) { return mapDateField(field); - } else if (field.type() == IndexField.Type.NUMERIC) { - return mapNumericField(field); + } else if (field.type() == IndexField.Type.DOUBLE) { + return mapDoubleField(field); + } else if (field.type() == IndexField.Type.INTEGER) { + return mapIntegerField(field); + } else if (field.type() == IndexField.Type.LONG) { + return mapLongField(field); } else if (field.type() == IndexField.Type.UUID_PATH) { return mapUuidPathField(field); } else { @@ -284,10 +288,18 @@ public abstract class BaseIndex, KEY extends Serial "analyzer", "uuid_analyzer"); } - protected Map mapNumericField(IndexField field) { + protected Map mapDoubleField(IndexField field) { return ImmutableMap.of("type", "double"); } + protected Map mapIntegerField(IndexField field) { + return ImmutableMap.of("type", "integer"); + } + + protected Map mapLongField(IndexField field) { + return ImmutableMap.of("type", "long"); + } + protected Map mapBooleanField(IndexField field) { return ImmutableMap.of("type", "boolean"); } @@ -315,14 +327,7 @@ public abstract class BaseIndex, KEY extends Serial protected boolean needMultiField(IndexField field) { return (field.type() == IndexField.Type.TEXT || field.type() == IndexField.Type.STRING) - && (field.sortable() || field.searchable()); - } - - protected Map mapSortField(IndexField field) { - return ImmutableMap.of( - "type", "string", - "index", "analyzed", - "analyzer", "sortable"); + && (field.isSortable() || field.isSearchable()); } protected Map mapGramsField(IndexField field) { @@ -343,10 +348,13 @@ public abstract class BaseIndex, KEY extends Serial protected Map mapMultiField(IndexField field) { Map mapping = new HashMap(); - if (field.sortable()) { - mapping.put(IndexField.SORT_SUFFIX, mapSortField(field)); + if (field.isSortable()) { + mapping.put(IndexField.SORT_SUFFIX, ImmutableMap.of( + "type", "string", + "index", "analyzed", + "analyzer", "sortable")); } - if (field.searchable()) { + if (field.isSearchable()) { if (field.type() != IndexField.Type.TEXT) { mapping.put(IndexField.SEARCH_PARTIAL_SUFFIX, mapGramsField(field)); } @@ -509,16 +517,14 @@ public abstract class BaseIndex, KEY extends Serial Multimap stats = ArrayListMultimap.create(); if (aggregations != null) { for (Aggregation aggregation : aggregations.asList()) { - if (aggregation.getClass().isAssignableFrom(StringTerms.class)) { + if (aggregation instanceof StringTerms) { for (Terms.Bucket value : ((Terms) aggregation).getBuckets()) { - - FacetValue facetValue = new FacetValue(value.getKey(), (int) value.getDocCount()); - facetValue.setSubFacets(processAggregations(value.getAggregations())); + FacetValue facetValue = new FacetValue(value.getKey(), value.getDocCount()); stats.put(aggregation.getName(), facetValue); } - } else if (aggregation.getClass().isAssignableFrom(InternalValueCount.class)) { + } else if (aggregation instanceof InternalValueCount) { InternalValueCount count = (InternalValueCount) aggregation; - FacetValue facetValue = new FacetValue(count.getName(), (int) count.getValue()); + FacetValue facetValue = new FacetValue(count.getName(), count.getValue()); stats.put(count.getName(), facetValue); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/FacetValue.java b/server/sonar-server/src/main/java/org/sonar/server/search/FacetValue.java index 57dbe6f3a82..8ff062563e0 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/search/FacetValue.java +++ b/server/sonar-server/src/main/java/org/sonar/server/search/FacetValue.java @@ -19,57 +19,42 @@ */ package org.sonar.server.search; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.Multimap; -import org.apache.commons.lang.builder.EqualsBuilder; -import org.apache.commons.lang.builder.HashCodeBuilder; - public class FacetValue { - private String key; - private Integer value; - private Multimap subFacets; + private final String key; + private final long value; - public FacetValue(String key, Integer value){ + public FacetValue(String key, long value) { this.key = key; this.value = value; - this.subFacets = ArrayListMultimap.create(); } public String getKey() { return key; } - public FacetValue setKey(String key) { - this.key = key; - return this; - } - - public Integer getValue() { + public long getValue() { return value; } - public FacetValue setValue(Integer value) { - this.value = value; - return this; - } - - public Multimap getSubFacets() { - return subFacets; - } - - public FacetValue setSubFacets(Multimap subFacets) { - this.subFacets = subFacets; - return this; - } - @Override - public boolean equals(Object other) { - return EqualsBuilder.reflectionEquals(this, other); + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + FacetValue that = (FacetValue) o; + if (key != null ? !key.equals(that.key) : that.key != null) { + return false; + } + return true; } @Override public int hashCode() { - return HashCodeBuilder.reflectionHashCode(this); + return key != null ? key.hashCode() : 0; } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/IndexField.java b/server/sonar-server/src/main/java/org/sonar/server/search/IndexField.java index 5a271bfa061..ef171748b3a 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/search/IndexField.java +++ b/server/sonar-server/src/main/java/org/sonar/server/search/IndexField.java @@ -28,7 +28,7 @@ import java.util.Collections; public class IndexField { public static enum Type { - STRING, TEXT, DATE, BOOLEAN, NUMERIC, OBJECT, UUID_PATH + STRING, TEXT, DATE, BOOLEAN, INTEGER, LONG, DOUBLE, OBJECT, UUID_PATH } public static final String SORT_SUFFIX = "sort"; @@ -52,20 +52,20 @@ public class IndexField { this.nestedFields = nestedFields; } - public boolean sortable() { + public boolean isSortable() { return sortable; } - public IndexField sortable(boolean b) { + public IndexField setSortable(boolean b) { this.sortable = b; return this; } - public boolean searchable() { + public boolean isSearchable() { return searchable; } - public IndexField searchable(boolean b) { + public IndexField setSearchable(boolean b) { this.searchable = b; return this; } @@ -79,15 +79,15 @@ public class IndexField { } public Collection nestedFields() { - return this.nestedFields; + return nestedFields; } public String sortField() { - if (this.sortable()) { + if (isSortable()) { return this.field + ((type == IndexField.Type.TEXT || type == IndexField.Type.STRING) ? "." + IndexField.SORT_SUFFIX : ""); } else { - throw new IllegalStateException("Cannot sort on field (Field is not sortable)"); + throw new IllegalStateException("Field is not sortable: " + field); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/Indexable.java b/server/sonar-server/src/main/java/org/sonar/server/search/Indexable.java index 46846a6d51e..eb993d1feb3 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/search/Indexable.java +++ b/server/sonar-server/src/main/java/org/sonar/server/search/Indexable.java @@ -26,9 +26,6 @@ import java.util.Collection; import java.util.HashSet; import java.util.Set; -/** - * @since 4.4 - */ public class Indexable { private static final Set ALL_FIELDS = new HashSet(); @@ -48,7 +45,7 @@ public class Indexable { public static IndexField addSearchable(IndexField.Type type, String field){ IndexField indexField = new IndexField(type, field) - .searchable(true); + .setSearchable(true); ALL_FIELDS.add(indexField); return indexField; } @@ -56,15 +53,15 @@ public class Indexable { public static IndexField addSortableAndSearchable(IndexField.Type type, String field) { IndexField indexField = new IndexField(type, field) - .searchable(true) - .sortable(true); + .setSearchable(true) + .setSortable(true); ALL_FIELDS.add(indexField); return indexField; } public static IndexField addSortable(IndexField.Type type, String field){ IndexField indexField = new IndexField(type, field) - .sortable(true); + .setSortable(true); ALL_FIELDS.add(indexField); return indexField; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/Result.java b/server/sonar-server/src/main/java/org/sonar/server/search/Result.java index d7861e0cd85..844f31211fc 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/search/Result.java +++ b/server/sonar-server/src/main/java/org/sonar/server/search/Result.java @@ -72,14 +72,14 @@ public class Result { private void processAggregation(Aggregation aggregation) { if (Missing.class.isAssignableFrom(aggregation.getClass())) { Missing missing = (Missing) aggregation; - int docCount = (int) missing.getDocCount(); - if (docCount > 0) { + long docCount = missing.getDocCount(); + if (docCount > 0L) { this.facets.put(aggregation.getName().replace("_missing",""), new FacetValue("", docCount)); } } else if (Terms.class.isAssignableFrom(aggregation.getClass())) { Terms termAggregation = (Terms) aggregation; for (Terms.Bucket value : termAggregation.getBuckets()) { - this.facets.put(aggregation.getName().replace("_selected",""), new FacetValue(value.getKey(), (int) value.getDocCount())); + this.facets.put(aggregation.getName().replace("_selected",""), new FacetValue(value.getKey(), value.getDocCount())); } } else if (HasAggregations.class.isAssignableFrom(aggregation.getClass())) { HasAggregations hasAggregations = (HasAggregations) aggregation; diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java index ee6d3fa63e8..93104c738ee 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java @@ -400,12 +400,12 @@ public class IssueServiceMediumTest { IssueTesting.newDto(rule, file, project).setAssignee("steph")); session.commit(); - Map results = service.findIssueAssignees(IssueQuery.builder().build()); + Map results = service.findIssueAssignees(IssueQuery.builder().build()); assertThat(results).hasSize(3); - assertThat(results.get("steph")).isEqualTo(2); - assertThat(results.get("simon")).isEqualTo(1); - assertThat(results.get(null)).isEqualTo(1); + assertThat(results.get("steph")).isEqualTo(2L); + assertThat(results.get("simon")).isEqualTo(1L); + assertThat(results.get(null)).isEqualTo(1L); assertThat(results.keySet().toArray()[0]).isEqualTo("steph"); assertThat(results.keySet().toArray()[1]).isEqualTo("simon"); diff --git a/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexMediumTest.java index 627314159fd..dc0dd4c16f9 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexMediumTest.java @@ -55,11 +55,9 @@ import static org.fest.assertions.Fail.fail; public class RuleIndexMediumTest extends SearchMediumTest { - RuleDao dao = tester.get(RuleDao.class); RuleIndex index = tester.get(RuleIndex.class); - @Test public void getByKey() throws InterruptedException { RuleDto ruleDto = RuleTesting.newDto(RuleKey.of("javascript", "S001")); @@ -71,9 +69,6 @@ public class RuleIndexMediumTest extends SearchMediumTest { assertThat(rule.htmlDescription()).isEqualTo(ruleDto.getDescription()); assertThat(rule.key()).isEqualTo(ruleDto.getKey()); - // TODO - // assertThat(rule.debtSubCharacteristicKey()) - // .isEqualTo(ruleDto.getDefaultSubCharacteristicId().toString()); assertThat(rule.debtRemediationFunction().type().name()) .isEqualTo(ruleDto.getRemediationFunction()); @@ -551,7 +546,7 @@ public class RuleIndexMediumTest extends SearchMediumTest { new RuleQuery().setSortField(RuleNormalizer.RuleField.LANGUAGE); fail(); } catch (IllegalStateException e) { - assertThat(e.getMessage()).isEqualTo("Field 'lang' is not sortable!"); + assertThat(e).hasMessage("Field 'lang' is not sortable"); } }