DAOv.2 - Updated tag for aggregation in RuleService and Normalizer

6 files changed, 43 insertions, 50 deletions

diff --git a/sonar-server/src/main/java/org/sonar/server/rule2/RuleService.java b/sonar-server/src/main/java/org/sonar/server/rule2/RuleService.java
index a9c4bf867d1..340ae3e07a6 100644
--- a/sonar-server/src/main/java/org/sonar/server/rule2/RuleService.java
+++ b/sonar-server/src/main/java/org/sonar/server/rule2/RuleService.java
@@ -97,9 +97,16 @@ public class RuleService implements ServerComponent {
    * List all tags, including system tags, defined on rules
    */
   public Set<String> listTags() {
-    return index.terms(RuleNormalizer.RuleField.TAGS.key(), RuleNormalizer.RuleField.SYSTEM_TAGS.key());
+    /** using combined _TAGS field of ES until ES update that has multiTerms aggregation */
+    return index.terms(RuleNormalizer.RuleField._TAGS.key());
   }
 
+  /**
+   * Set tags for rule.
+   *
+   * @param ruleKey  the required key
+   * @param tags     Set of tags. <code>null</code> to remove all tags.
+   */
   public void setTags(RuleKey ruleKey, Set<String> tags) {
 
     checkAdminPermission(UserSession.get());
diff --git a/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleIndex.java b/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleIndex.java
index eed0f30537a..4ca8faf2b69 100644
--- a/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleIndex.java
+++ b/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleIndex.java
@@ -31,8 +31,6 @@ import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
-import org.elasticsearch.search.facet.FacetBuilders;
-import org.elasticsearch.search.facet.terms.TermsFacet;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
@@ -147,8 +145,14 @@ public class RuleIndex extends BaseIndex<Rule, RuleDto, RuleKey> {
 
     addMatchField(mapping, RuleNormalizer.RuleField.LANGUAGE.key(), "string");
 
+    mapping.startObject(RuleNormalizer.RuleField._TAGS.key())
+      .field("type", "string")
+      .field("analyzer", "whitespace")
+      .endObject();
+
     mapping.startObject(RuleNormalizer.RuleField.TAGS.key())
       .field("type", "string")
+      .field("analyzer", "whitespace")
       .endObject();
 
     mapping.startObject(RuleNormalizer.RuleField.SYSTEM_TAGS.key())
@@ -275,7 +279,7 @@ public class RuleIndex extends BaseIndex<Rule, RuleDto, RuleKey> {
         RuleNormalizer.RuleField.KEY.key(),
         RuleNormalizer.RuleField.KEY.key() + ".search",
         RuleNormalizer.RuleField.LANGUAGE.key(),
-        RuleNormalizer.RuleField.TAGS.key());
+        RuleNormalizer.RuleField._TAGS.key());
     } else {
       qb = QueryBuilders.matchAllQuery();
     }
@@ -289,8 +293,8 @@ public class RuleIndex extends BaseIndex<Rule, RuleDto, RuleKey> {
     this.addTermFilter(RuleNormalizer.RuleField.REPOSITORY.key(), query.getRepositories(), fb);
     this.addTermFilter(RuleNormalizer.RuleField.SEVERITY.key(), query.getSeverities(), fb);
     this.addTermFilter(RuleNormalizer.RuleField.KEY.key(), query.getKey(), fb);
+    this.addTermFilter(RuleNormalizer.RuleField._TAGS.key(), query.getTags(), fb);
 
-    this.addMultiFieldTermFilter(query.getTags(), fb, RuleNormalizer.RuleField.TAGS.key(), RuleNormalizer.RuleField.SYSTEM_TAGS.key());
 
     if (query.getStatuses() != null && !query.getStatuses().isEmpty()) {
       Collection<String> stringStatus = new ArrayList<String>();
@@ -349,8 +353,7 @@ public class RuleIndex extends BaseIndex<Rule, RuleDto, RuleKey> {
      /* the Tag facet */
     query.addAggregation(AggregationBuilders
       .terms("Tags")
-      .field(RuleNormalizer.RuleField.SYSTEM_TAGS.key())
-      .field(RuleNormalizer.RuleField.TAGS.key())
+      .field(RuleNormalizer.RuleField._TAGS.key())
       .order(Terms.Order.count(false))
       .size(10)
       .minDocCount(0));
@@ -384,25 +387,25 @@ public class RuleIndex extends BaseIndex<Rule, RuleDto, RuleKey> {
     return new RuleDoc(fields);
   }
 
-  public Set<String> terms(String... fields) {
+
+  public Set<String> terms(String fields) {
     Set<String> tags = new HashSet<String>();
+    String key = "_ref";
 
     SearchRequestBuilder request = this.getClient()
       .prepareSearch(this.getIndexName())
       .setQuery(QueryBuilders.matchAllQuery())
-      .addFacet(FacetBuilders.termsFacet("tags")
-        .allTerms(false)
-        .fields(fields)
-        .global(true)
-        .size(Integer.MAX_VALUE));
+      .addAggregation(AggregationBuilders.terms(key)
+        .field(fields)
+        .size(Integer.MAX_VALUE)
+        .minDocCount(1));
 
     SearchResponse esResponse = request.get();
 
-    TermsFacet termFacet = esResponse
-      .getFacets().facet("tags");
+    Terms aggregation = (Terms) esResponse.getAggregations().get(key);
 
-    for (TermsFacet.Entry facetValue : termFacet.getEntries()) {
-      tags.add(facetValue.getTerm().string());
+    for (Terms.Bucket value : aggregation.getBuckets()){
+      tags.add(value.getKey());
     }
     return tags;
   }
diff --git a/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleNormalizer.java b/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleNormalizer.java
index e0b2850efdd..6c39c5292e5 100644
--- a/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleNormalizer.java
+++ b/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleNormalizer.java
@@ -23,6 +23,7 @@ import com.google.common.base.Function;
 import com.google.common.collect.Collections2;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.sonar.api.rule.RuleKey;
 import org.sonar.check.Cardinality;
@@ -64,7 +65,8 @@ public class RuleNormalizer extends BaseNormalizer<RuleDto, RuleKey> {
     NOTE("markdownNote"),
     NOTE_LOGIN("noteLogin"),
     NOTE_CREATED_AT("noteCreatedAt"),
-    NOTE_UPDATED_AT("noteUpdatedAt");
+    NOTE_UPDATED_AT("noteUpdatedAt"),
+    _TAGS("_tags");
 
     private final String key;
 
@@ -128,6 +130,7 @@ public class RuleNormalizer extends BaseNormalizer<RuleDto, RuleKey> {
   @Override
   public UpdateRequest normalize(RuleDto rule) {
 
+    /** Update Fields */
     Map<String, Object> update = new HashMap<String, Object>();
     update.put(RuleField.KEY.key(), rule.getKey().toString());
     update.put(RuleField.REPOSITORY.key(), rule.getRepositoryKey());
@@ -176,19 +179,18 @@ public class RuleNormalizer extends BaseNormalizer<RuleDto, RuleKey> {
     update.put(RuleField.DEBT_FUNCTION_TYPE.key(), dType);
     update.put(RuleField.DEBT_FUNCTION_COEFFICIENT.key(), dCoefficient);
     update.put(RuleField.DEBT_FUNCTION_OFFSET.key(), dOffset);
-
     update.put(RuleField.TAGS.key(), rule.getTags());
     update.put(RuleField.SYSTEM_TAGS.key(), rule.getSystemTags());
+    update.put(RuleField._TAGS.key(), Sets.union(rule.getSystemTags(), rule.getTags()));
 
 
-
-      /* Upsert elements */
+    /** Upsert elements */
     Map<String, Object> upsert = new HashMap<String, Object>(update);
     upsert.put(RuleField.KEY.key(), rule.getKey().toString());
     upsert.put(RuleField.PARAMS.key(), new ArrayList<String>());
 
 
-    /* Creating updateRequest */
+    /** Creating updateRequest */
     return new UpdateRequest()
       .doc(update)
       .upsert(upsert);
diff --git a/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java b/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java
index 635944fc6ca..7a68cd2cdd7 100644
--- a/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java
+++ b/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java
@@ -39,7 +39,6 @@ import org.sonar.server.es.ESNode;
 
 import java.io.IOException;
 import java.io.Serializable;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
@@ -279,29 +278,6 @@ public abstract class BaseIndex<D, E extends Dto<K>, K extends Serializable>
       .endObject();
   }
 
-  protected void addFindField(XContentBuilder mapping, String field, String type) throws IOException {
-    mapping.startObject(field)
-      .field("type", type)
-      .field("index", "analyzed")
-      .endObject();
-  }
-
-
-  protected BoolFilterBuilder addMultiFieldTermFilter(Collection<String> values, BoolFilterBuilder filter, String... fields) {
-    if (values != null && !values.isEmpty()) {
-      BoolFilterBuilder valuesFilter = FilterBuilders.boolFilter();
-      for (String value : values) {
-        Collection<FilterBuilder> filterBuilders = new ArrayList<FilterBuilder>();
-        for (String field : fields) {
-          filterBuilders.add(FilterBuilders.termFilter(field, value));
-        }
-        valuesFilter.should(FilterBuilders.orFilter(filterBuilders.toArray(new FilterBuilder[filterBuilders.size()])));
-      }
-      filter.must(valuesFilter);
-    }
-    return filter;
-  }
-
 
   protected BoolFilterBuilder addTermFilter(String field, Collection<String> values, BoolFilterBuilder filter) {
     if (values != null && !values.isEmpty()) {
diff --git a/sonar-server/src/test/java/org/sonar/server/rule2/RuleServiceMediumTest.java b/sonar-server/src/test/java/org/sonar/server/rule2/RuleServiceMediumTest.java
index 4c5e21aebed..cf691a4a281 100644
--- a/sonar-server/src/test/java/org/sonar/server/rule2/RuleServiceMediumTest.java
+++ b/sonar-server/src/test/java/org/sonar/server/rule2/RuleServiceMediumTest.java
@@ -180,8 +180,7 @@ public class RuleServiceMediumTest {
 
 
 
-    Set<String> tags = index.terms(RuleNormalizer.RuleField.TAGS.key(),
-      RuleNormalizer.RuleField.SYSTEM_TAGS.key());
+    Set<String> tags = index.terms(RuleNormalizer.RuleField._TAGS.key());
     assertThat(tags).containsOnly("java-coding","security",
       "stephane.gamard@sonarsource.com","mytag");
 
diff --git a/sonar-server/src/test/java/org/sonar/server/rule2/index/RuleIndexMediumTest.java b/sonar-server/src/test/java/org/sonar/server/rule2/index/RuleIndexMediumTest.java
index c235d15878f..9baa5f01688 100644
--- a/sonar-server/src/test/java/org/sonar/server/rule2/index/RuleIndexMediumTest.java
+++ b/sonar-server/src/test/java/org/sonar/server/rule2/index/RuleIndexMediumTest.java
@@ -103,7 +103,8 @@ public class RuleIndexMediumTest {
   @Test
   public void global_facet_on_repositories() {
     dao.insert(newRuleDto(RuleKey.of("javascript", "S001")).setRuleKey("X001"), dbSession);
-    dao.insert(newRuleDto(RuleKey.of("php", "S001")), dbSession);
+    dao.insert(newRuleDto(RuleKey.of("php", "S001"))
+      .setSystemTags(ImmutableSet.of("sysTag")), dbSession);
     dao.insert(newRuleDto(RuleKey.of("javascript", "S002")).setRuleKey("X002")
       .setTags(ImmutableSet.of("tag1")), dbSession);
     dbSession.commit();
@@ -116,15 +117,20 @@ public class RuleIndexMediumTest {
 
     // Repositories Facet is preset
     result = index.search(query, new QueryOptions().setFacet(true));
-
     assertThat(result.getFacets()).isNotNull();
     assertThat(result.getFacets()).hasSize(3);
+
+    // Verify the value of a given facet
     Collection<FacetValue> repoFacets = result.getFacetValues("Repositories");
     assertThat(repoFacets).hasSize(2);
     assertThat(Iterables.get(repoFacets, 0).getKey()).isEqualTo("javascript");
     assertThat(Iterables.get(repoFacets, 0).getValue()).isEqualTo(2);
     assertThat(Iterables.get(repoFacets, 1).getKey()).isEqualTo("php");
     assertThat(Iterables.get(repoFacets, 1).getValue()).isEqualTo(1);
+
+    // Check that tag facet has both Tags and SystemTags values
+    Collection<FacetValue> tagFacet = result.getFacetValues("Tags");
+    assertThat(tagFacet).hasSize(2);
   }
 
   @Test