From 44d579f56005ef7fb0c6277e09c368ada7484ca8 Mon Sep 17 00:00:00 2001 From: Stephane Gamard Date: Thu, 22 May 2014 17:46:07 +0200 Subject: [PATCH] DAOv.2 - Updated tag for aggregation in RuleService and Normalizer --- .../org/sonar/server/rule2/RuleService.java | 9 ++++- .../sonar/server/rule2/index/RuleIndex.java | 35 ++++++++++--------- .../server/rule2/index/RuleNormalizer.java | 12 ++++--- .../org/sonar/server/search/BaseIndex.java | 24 ------------- .../server/rule2/RuleServiceMediumTest.java | 3 +- .../rule2/index/RuleIndexMediumTest.java | 10 ++++-- 6 files changed, 43 insertions(+), 50 deletions(-) diff --git a/sonar-server/src/main/java/org/sonar/server/rule2/RuleService.java b/sonar-server/src/main/java/org/sonar/server/rule2/RuleService.java index a9c4bf867d1..340ae3e07a6 100644 --- a/sonar-server/src/main/java/org/sonar/server/rule2/RuleService.java +++ b/sonar-server/src/main/java/org/sonar/server/rule2/RuleService.java @@ -97,9 +97,16 @@ public class RuleService implements ServerComponent { * List all tags, including system tags, defined on rules */ public Set listTags() { - return index.terms(RuleNormalizer.RuleField.TAGS.key(), RuleNormalizer.RuleField.SYSTEM_TAGS.key()); + /** using combined _TAGS field of ES until ES update that has multiTerms aggregation */ + return index.terms(RuleNormalizer.RuleField._TAGS.key()); } + /** + * Set tags for rule. + * + * @param ruleKey the required key + * @param tags Set of tags. null to remove all tags. + */ public void setTags(RuleKey ruleKey, Set tags) { checkAdminPermission(UserSession.get()); diff --git a/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleIndex.java b/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleIndex.java index eed0f30537a..4ca8faf2b69 100644 --- a/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleIndex.java +++ b/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleIndex.java @@ -31,8 +31,6 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.facet.FacetBuilders; -import org.elasticsearch.search.facet.terms.TermsFacet; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; @@ -147,8 +145,14 @@ public class RuleIndex extends BaseIndex { addMatchField(mapping, RuleNormalizer.RuleField.LANGUAGE.key(), "string"); + mapping.startObject(RuleNormalizer.RuleField._TAGS.key()) + .field("type", "string") + .field("analyzer", "whitespace") + .endObject(); + mapping.startObject(RuleNormalizer.RuleField.TAGS.key()) .field("type", "string") + .field("analyzer", "whitespace") .endObject(); mapping.startObject(RuleNormalizer.RuleField.SYSTEM_TAGS.key()) @@ -275,7 +279,7 @@ public class RuleIndex extends BaseIndex { RuleNormalizer.RuleField.KEY.key(), RuleNormalizer.RuleField.KEY.key() + ".search", RuleNormalizer.RuleField.LANGUAGE.key(), - RuleNormalizer.RuleField.TAGS.key()); + RuleNormalizer.RuleField._TAGS.key()); } else { qb = QueryBuilders.matchAllQuery(); } @@ -289,8 +293,8 @@ public class RuleIndex extends BaseIndex { this.addTermFilter(RuleNormalizer.RuleField.REPOSITORY.key(), query.getRepositories(), fb); this.addTermFilter(RuleNormalizer.RuleField.SEVERITY.key(), query.getSeverities(), fb); this.addTermFilter(RuleNormalizer.RuleField.KEY.key(), query.getKey(), fb); + this.addTermFilter(RuleNormalizer.RuleField._TAGS.key(), query.getTags(), fb); - this.addMultiFieldTermFilter(query.getTags(), fb, RuleNormalizer.RuleField.TAGS.key(), RuleNormalizer.RuleField.SYSTEM_TAGS.key()); if (query.getStatuses() != null && !query.getStatuses().isEmpty()) { Collection stringStatus = new ArrayList(); @@ -349,8 +353,7 @@ public class RuleIndex extends BaseIndex { /* the Tag facet */ query.addAggregation(AggregationBuilders .terms("Tags") - .field(RuleNormalizer.RuleField.SYSTEM_TAGS.key()) - .field(RuleNormalizer.RuleField.TAGS.key()) + .field(RuleNormalizer.RuleField._TAGS.key()) .order(Terms.Order.count(false)) .size(10) .minDocCount(0)); @@ -384,25 +387,25 @@ public class RuleIndex extends BaseIndex { return new RuleDoc(fields); } - public Set terms(String... fields) { + + public Set terms(String fields) { Set tags = new HashSet(); + String key = "_ref"; SearchRequestBuilder request = this.getClient() .prepareSearch(this.getIndexName()) .setQuery(QueryBuilders.matchAllQuery()) - .addFacet(FacetBuilders.termsFacet("tags") - .allTerms(false) - .fields(fields) - .global(true) - .size(Integer.MAX_VALUE)); + .addAggregation(AggregationBuilders.terms(key) + .field(fields) + .size(Integer.MAX_VALUE) + .minDocCount(1)); SearchResponse esResponse = request.get(); - TermsFacet termFacet = esResponse - .getFacets().facet("tags"); + Terms aggregation = (Terms) esResponse.getAggregations().get(key); - for (TermsFacet.Entry facetValue : termFacet.getEntries()) { - tags.add(facetValue.getTerm().string()); + for (Terms.Bucket value : aggregation.getBuckets()){ + tags.add(value.getKey()); } return tags; } diff --git a/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleNormalizer.java b/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleNormalizer.java index e0b2850efdd..6c39c5292e5 100644 --- a/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleNormalizer.java +++ b/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleNormalizer.java @@ -23,6 +23,7 @@ import com.google.common.base.Function; import com.google.common.collect.Collections2; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import org.elasticsearch.action.update.UpdateRequest; import org.sonar.api.rule.RuleKey; import org.sonar.check.Cardinality; @@ -64,7 +65,8 @@ public class RuleNormalizer extends BaseNormalizer { NOTE("markdownNote"), NOTE_LOGIN("noteLogin"), NOTE_CREATED_AT("noteCreatedAt"), - NOTE_UPDATED_AT("noteUpdatedAt"); + NOTE_UPDATED_AT("noteUpdatedAt"), + _TAGS("_tags"); private final String key; @@ -128,6 +130,7 @@ public class RuleNormalizer extends BaseNormalizer { @Override public UpdateRequest normalize(RuleDto rule) { + /** Update Fields */ Map update = new HashMap(); update.put(RuleField.KEY.key(), rule.getKey().toString()); update.put(RuleField.REPOSITORY.key(), rule.getRepositoryKey()); @@ -176,19 +179,18 @@ public class RuleNormalizer extends BaseNormalizer { update.put(RuleField.DEBT_FUNCTION_TYPE.key(), dType); update.put(RuleField.DEBT_FUNCTION_COEFFICIENT.key(), dCoefficient); update.put(RuleField.DEBT_FUNCTION_OFFSET.key(), dOffset); - update.put(RuleField.TAGS.key(), rule.getTags()); update.put(RuleField.SYSTEM_TAGS.key(), rule.getSystemTags()); + update.put(RuleField._TAGS.key(), Sets.union(rule.getSystemTags(), rule.getTags())); - - /* Upsert elements */ + /** Upsert elements */ Map upsert = new HashMap(update); upsert.put(RuleField.KEY.key(), rule.getKey().toString()); upsert.put(RuleField.PARAMS.key(), new ArrayList()); - /* Creating updateRequest */ + /** Creating updateRequest */ return new UpdateRequest() .doc(update) .upsert(upsert); diff --git a/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java b/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java index 635944fc6ca..7a68cd2cdd7 100644 --- a/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java +++ b/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java @@ -39,7 +39,6 @@ import org.sonar.server.es.ESNode; import java.io.IOException; import java.io.Serializable; -import java.util.ArrayList; import java.util.Collection; import java.util.Map; import java.util.concurrent.ExecutionException; @@ -279,29 +278,6 @@ public abstract class BaseIndex, K extends Serializable> .endObject(); } - protected void addFindField(XContentBuilder mapping, String field, String type) throws IOException { - mapping.startObject(field) - .field("type", type) - .field("index", "analyzed") - .endObject(); - } - - - protected BoolFilterBuilder addMultiFieldTermFilter(Collection values, BoolFilterBuilder filter, String... fields) { - if (values != null && !values.isEmpty()) { - BoolFilterBuilder valuesFilter = FilterBuilders.boolFilter(); - for (String value : values) { - Collection filterBuilders = new ArrayList(); - for (String field : fields) { - filterBuilders.add(FilterBuilders.termFilter(field, value)); - } - valuesFilter.should(FilterBuilders.orFilter(filterBuilders.toArray(new FilterBuilder[filterBuilders.size()]))); - } - filter.must(valuesFilter); - } - return filter; - } - protected BoolFilterBuilder addTermFilter(String field, Collection values, BoolFilterBuilder filter) { if (values != null && !values.isEmpty()) { diff --git a/sonar-server/src/test/java/org/sonar/server/rule2/RuleServiceMediumTest.java b/sonar-server/src/test/java/org/sonar/server/rule2/RuleServiceMediumTest.java index 4c5e21aebed..cf691a4a281 100644 --- a/sonar-server/src/test/java/org/sonar/server/rule2/RuleServiceMediumTest.java +++ b/sonar-server/src/test/java/org/sonar/server/rule2/RuleServiceMediumTest.java @@ -180,8 +180,7 @@ public class RuleServiceMediumTest { - Set tags = index.terms(RuleNormalizer.RuleField.TAGS.key(), - RuleNormalizer.RuleField.SYSTEM_TAGS.key()); + Set tags = index.terms(RuleNormalizer.RuleField._TAGS.key()); assertThat(tags).containsOnly("java-coding","security", "stephane.gamard@sonarsource.com","mytag"); diff --git a/sonar-server/src/test/java/org/sonar/server/rule2/index/RuleIndexMediumTest.java b/sonar-server/src/test/java/org/sonar/server/rule2/index/RuleIndexMediumTest.java index c235d15878f..9baa5f01688 100644 --- a/sonar-server/src/test/java/org/sonar/server/rule2/index/RuleIndexMediumTest.java +++ b/sonar-server/src/test/java/org/sonar/server/rule2/index/RuleIndexMediumTest.java @@ -103,7 +103,8 @@ public class RuleIndexMediumTest { @Test public void global_facet_on_repositories() { dao.insert(newRuleDto(RuleKey.of("javascript", "S001")).setRuleKey("X001"), dbSession); - dao.insert(newRuleDto(RuleKey.of("php", "S001")), dbSession); + dao.insert(newRuleDto(RuleKey.of("php", "S001")) + .setSystemTags(ImmutableSet.of("sysTag")), dbSession); dao.insert(newRuleDto(RuleKey.of("javascript", "S002")).setRuleKey("X002") .setTags(ImmutableSet.of("tag1")), dbSession); dbSession.commit(); @@ -116,15 +117,20 @@ public class RuleIndexMediumTest { // Repositories Facet is preset result = index.search(query, new QueryOptions().setFacet(true)); - assertThat(result.getFacets()).isNotNull(); assertThat(result.getFacets()).hasSize(3); + + // Verify the value of a given facet Collection repoFacets = result.getFacetValues("Repositories"); assertThat(repoFacets).hasSize(2); assertThat(Iterables.get(repoFacets, 0).getKey()).isEqualTo("javascript"); assertThat(Iterables.get(repoFacets, 0).getValue()).isEqualTo(2); assertThat(Iterables.get(repoFacets, 1).getKey()).isEqualTo("php"); assertThat(Iterables.get(repoFacets, 1).getValue()).isEqualTo(1); + + // Check that tag facet has both Tags and SystemTags values + Collection tagFacet = result.getFacetValues("Tags"); + assertThat(tagFacet).hasSize(2); } @Test -- 2.39.5