From d4087f1c01ead9ecc56222e6795406f36dfa0cf1 Mon Sep 17 00:00:00 2001 From: Julien Lancelot Date: Thu, 18 Feb 2016 15:33:08 +0100 Subject: [PATCH] SONAR-7330 Add rule index definition --- .../java/org/sonar/server/es/BaseIndex.java | 5 + .../java/org/sonar/server/es/EsUtils.java | 49 +- .../java/org/sonar/server/es/NewIndex.java | 21 +- .../org/sonar/server/es/SearchIdResult.java | 66 ++ .../sonar/server/issue/index/IssueIndex.java | 6 +- .../server/issue/index/IssueIndexer.java | 2 +- .../platformlevel/PlatformLevel1.java | 2 + .../platformlevel/PlatformLevel4.java | 4 + .../qualityprofile/index/ActiveRuleIndex.java | 21 +- .../org/sonar/server/rule/RuleService.java | 26 +- .../org/sonar/server/rule/index/RuleDoc.java | 133 ++- .../sonar/server/rule/index/RuleIndex.java | 2 +- .../sonar/server/rule/index/RuleIndex2.java | 493 +++++++++ .../rule/index/RuleIndexDefinition.java | 100 ++ .../sonar/server/rule/index/RuleIndexer.java | 101 ++ .../sonar/server/rule/index/RuleQuery.java | 38 +- .../rule/index/RuleResultSetIterator.java | 141 +++ .../sonar/server/rule/ws/SearchAction.java | 56 +- .../server/search/IndexSynchronizer.java | 6 +- .../java/org/sonar/server/es/EsTester.java | 11 +- .../org/sonar/server/es/NewIndexTest.java | 25 +- .../server/rule/RuleBackendMediumTest.java | 418 -------- .../server/rule/index/RuleDocTesting.java | 50 + .../server/rule/index/RuleIndex2Test.java | 732 ++++++++++++++ .../rule/index/RuleIndexDefinitionTest.java | 48 + .../rule/index/RuleIndexMediumTest.java | 935 ------------------ .../server/rule/index/RuleIndexerTest.java | 109 ++ .../rule/index/RuleResultSetIteratorTest.java | 169 ++++ .../rule/db/RuleDaoTest/insert-result.xml | 2 +- .../rule/db/RuleDaoTest/insert_all-result.xml | 4 +- .../rule/db/RuleDaoTest/update-result.xml | 2 +- .../rule/index/RuleIndexerTest/index.xml | 13 + .../index/RuleIndexerTest/removed_rule.xml | 13 + .../RuleResultSetIteratorTest/custom_rule.xml | 25 + .../RuleResultSetIteratorTest/one_rule.xml | 13 + .../removed_rule.xml | 13 + .../RuleResultSetIteratorTest/shared.xml | 22 + .../FeedRulesLongDateColumnsTest/execute.xml | 2 +- 38 files changed, 2405 insertions(+), 1473 deletions(-) create mode 100644 server/sonar-server/src/main/java/org/sonar/server/es/SearchIdResult.java create mode 100644 server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex2.java create mode 100644 server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java create mode 100644 server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java create mode 100644 server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleResultSetIterator.java delete mode 100644 server/sonar-server/src/test/java/org/sonar/server/rule/RuleBackendMediumTest.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleDocTesting.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndex2Test.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexDefinitionTest.java delete mode 100644 server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexMediumTest.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleResultSetIteratorTest.java create mode 100644 server/sonar-server/src/test/resources/org/sonar/server/rule/index/RuleIndexerTest/index.xml create mode 100644 server/sonar-server/src/test/resources/org/sonar/server/rule/index/RuleIndexerTest/removed_rule.xml create mode 100644 server/sonar-server/src/test/resources/org/sonar/server/rule/index/RuleResultSetIteratorTest/custom_rule.xml create mode 100644 server/sonar-server/src/test/resources/org/sonar/server/rule/index/RuleResultSetIteratorTest/one_rule.xml create mode 100644 server/sonar-server/src/test/resources/org/sonar/server/rule/index/RuleResultSetIteratorTest/removed_rule.xml create mode 100644 server/sonar-server/src/test/resources/org/sonar/server/rule/index/RuleResultSetIteratorTest/shared.xml diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndex.java b/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndex.java index dc40fe90df0..1f7d44e3716 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndex.java @@ -23,6 +23,11 @@ import org.sonar.api.server.ServerSide; @ServerSide public abstract class BaseIndex { + + public static final String SORT_SUFFIX = "sort"; + public static final String SEARCH_WORDS_SUFFIX = "words"; + public static final String SEARCH_PARTIAL_SUFFIX = "grams"; + private final EsClient client; public BaseIndex(EsClient client) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java b/server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java index 46cd6e09347..ad081be93f4 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java @@ -21,17 +21,6 @@ package org.sonar.server.es; import com.google.common.base.Function; import com.google.common.collect.Lists; -import org.elasticsearch.action.search.SearchScrollRequestBuilder; -import org.elasticsearch.common.joda.time.format.ISODateTimeFormat; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.sonar.server.search.BaseDoc; - -import javax.annotation.CheckForNull; -import javax.annotation.Nullable; - import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; @@ -42,6 +31,15 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.Queue; +import javax.annotation.CheckForNull; +import javax.annotation.Nullable; +import org.elasticsearch.action.search.SearchScrollRequestBuilder; +import org.elasticsearch.common.joda.time.format.ISODateTimeFormat; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.sonar.server.search.BaseDoc; public class EsUtils { @@ -121,4 +119,33 @@ public class EsUtils { } }; } + + public static Iterator scrollIds(final EsClient esClient, final String scrollId, final Function idConverter) { + return new Iterator() { + private final Queue hits = new ArrayDeque<>(); + + @Override + public boolean hasNext() { + if (hits.isEmpty()) { + SearchScrollRequestBuilder esRequest = esClient.prepareSearchScroll(scrollId) + .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)); + Collections.addAll(hits, esRequest.get().getHits().getHits()); + } + return !hits.isEmpty(); + } + + @Override + public ID next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return idConverter.apply(hits.poll().getId()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException("Cannot remove item when scrolling"); + } + }; + } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java index 3a930d2937b..b68726aa9c9 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java @@ -54,7 +54,7 @@ public class NewIndex { // defaults attributes.put("dynamic", false); attributes.put("_all", ImmutableSortedMap.of("enabled", false)); - + attributes.put("_source", ImmutableSortedMap.of("enabled", true)); attributes.put("properties", properties); } @@ -78,6 +78,11 @@ public class NewIndex { return this; } + public NewIndexType setEnableSource(boolean enableSource) { + attributes.put("_source", ImmutableSortedMap.of("enabled", enableSource)); + return this; + } + public StringFieldBuilder stringFieldBuilder(String fieldName) { return new StringFieldBuilder(this, fieldName); } @@ -250,7 +255,13 @@ public class NewIndex { public void build() { validate(); Map hash = new TreeMap<>(); - if (!subFields.isEmpty()) { + if (subFields.isEmpty()) { + hash.putAll(ImmutableMap.of( + "type", "string", + "index", disableSearch ? "no" : "not_analyzed", + "omit_norms", "true", + "doc_values", docValues)); + } else { hash.put("type", "multi_field"); Map multiFields = new TreeMap<>(subFields); multiFields.put(fieldName, ImmutableMap.of( @@ -259,12 +270,6 @@ public class NewIndex { "omit_norms", "true", "doc_values", docValues)); hash.put("fields", multiFields); - } else { - hash.putAll(ImmutableMap.of( - "type", "string", - "index", disableSearch ? "no" : "not_analyzed", - "omit_norms", "true", - "doc_values", docValues)); } indexType.setProperty(fieldName, hash); diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/SearchIdResult.java b/server/sonar-server/src/main/java/org/sonar/server/es/SearchIdResult.java new file mode 100644 index 00000000000..7936a38e07e --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/es/SearchIdResult.java @@ -0,0 +1,66 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es; + +import com.google.common.base.Function; +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.lang.builder.ReflectionToStringBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; + +public class SearchIdResult { + + private final List ids; + private final Facets facets; + private final long total; + + public SearchIdResult(SearchResponse response, Function converter) { + this.facets = new Facets(response); + this.total = response.getHits().totalHits(); + this.ids = convertToIds(response.getHits(), converter); + } + + public List getIds() { + return ids; + } + + public long getTotal() { + return total; + } + + public Facets getFacets() { + return this.facets; + } + + @Override + public String toString() { + return ReflectionToStringBuilder.toString(this); + } + + public static List convertToIds(SearchHits hits, Function converter) { + List docs = new ArrayList<>(); + for (SearchHit hit : hits.getHits()) { + docs.add(converter.apply(hit.getId())); + } + return docs; + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java index ef3be38d935..37ac9f4af75 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java @@ -76,12 +76,12 @@ import org.sonar.server.es.SearchResult; import org.sonar.server.es.Sorting; import org.sonar.server.exceptions.NotFoundException; import org.sonar.server.issue.IssueQuery; -import org.sonarqube.ws.client.issue.IssueFilterParameters; -import org.sonar.server.rule.index.RuleNormalizer; +import org.sonar.server.rule.index.RuleIndexDefinition; import org.sonar.server.search.IndexDefinition; import org.sonar.server.search.StickyFacetBuilder; import org.sonar.server.user.UserSession; import org.sonar.server.view.index.ViewIndexDefinition; +import org.sonarqube.ws.client.issue.IssueFilterParameters; import static com.google.common.collect.Lists.newArrayList; @@ -652,7 +652,7 @@ public class IssueIndex extends BaseIndex { issueTags.include(String.format(SUBSTRING_MATCH_REGEXP, textQuery)); } TermsBuilder ruleTags = AggregationBuilders.terms(tagsOnRulesSubAggregation) - .field(RuleNormalizer.RuleField.ALL_TAGS.field()) + .field(RuleIndexDefinition.FIELD_RULE_ALL_TAGS) .size(maxNumberOfTags) .order(Terms.Order.term(true)) .minDocCount(1L); diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java index 48d17194db9..d6b21d28ba9 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java @@ -106,7 +106,7 @@ public class IssueIndexer extends BaseIndexer { bulk.stop(); } - BulkIndexer createBulkIndexer(boolean large) { + private BulkIndexer createBulkIndexer(boolean large) { BulkIndexer bulk = new BulkIndexer(esClient, IssueIndexDefinition.INDEX); bulk.setLarge(large); return bulk; diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel1.java b/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel1.java index 04dba264df4..6493e4d5c9c 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel1.java +++ b/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel1.java @@ -48,6 +48,7 @@ import org.sonar.server.qualityprofile.index.ActiveRuleNormalizer; import org.sonar.server.ruby.PlatformRackBridge; import org.sonar.server.rule.db.RuleDao; import org.sonar.server.rule.index.RuleIndex; +import org.sonar.server.rule.index.RuleIndex2; import org.sonar.server.rule.index.RuleNormalizer; import org.sonar.server.search.EsSearchModule; import org.sonar.server.search.IndexQueue; @@ -108,6 +109,7 @@ public class PlatformLevel1 extends PlatformLevel { ActiveRuleDao.class, // rules/qprofiles + RuleIndex2.class, RuleNormalizer.class, ActiveRuleNormalizer.class, RuleIndex.class, diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java b/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java index 9dfc23730e3..5bb7a911c2e 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java +++ b/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java @@ -256,6 +256,8 @@ import org.sonar.server.rule.RuleOperations; import org.sonar.server.rule.RuleRepositories; import org.sonar.server.rule.RuleService; import org.sonar.server.rule.RuleUpdater; +import org.sonar.server.rule.index.RuleIndexDefinition; +import org.sonar.server.rule.index.RuleIndexer; import org.sonar.server.rule.ws.ActiveRuleCompleter; import org.sonar.server.rule.ws.RepositoriesAction; import org.sonar.server.rule.ws.RuleMapper; @@ -430,6 +432,8 @@ public class PlatformLevel4 extends PlatformLevel { RubyQProfileActivityService.class, // rule + RuleIndexDefinition.class, + RuleIndexer.class, AnnotationRuleParser.class, XMLRuleParser.class, DefaultRuleFinder.class, diff --git a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java index b8b533bc7d9..b83cba83e3b 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java @@ -21,6 +21,11 @@ package org.sonar.server.qualityprofile.index; import com.google.common.collect.ImmutableList; import com.google.common.collect.Multimap; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; @@ -36,19 +41,13 @@ import org.sonar.api.rule.RuleStatus; import org.sonar.db.qualityprofile.ActiveRuleDto; import org.sonar.db.qualityprofile.ActiveRuleKey; import org.sonar.server.qualityprofile.ActiveRule; -import org.sonar.server.rule.index.RuleNormalizer; +import org.sonar.server.rule.index.RuleIndexDefinition; import org.sonar.server.search.BaseIndex; import org.sonar.server.search.FacetValue; import org.sonar.server.search.IndexDefinition; import org.sonar.server.search.IndexField; import org.sonar.server.search.SearchClient; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - public class ActiveRuleIndex extends BaseIndex { public static final String COUNT_ACTIVE_RULES = "countActiveRules"; @@ -139,7 +138,7 @@ public class ActiveRuleIndex extends BaseIndex countAllByQualityProfileKey() { return countByField(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY, FilterBuilders.hasParentFilter(IndexDefinition.RULE.getIndexType(), FilterBuilders.notFilter( - FilterBuilders.termFilter(RuleNormalizer.RuleField.STATUS.field(), "REMOVED")))); + FilterBuilders.termFilter(RuleIndexDefinition.FIELD_STATUS, "REMOVED")))); } public Multimap getStatsByProfileKey(String key) { @@ -174,7 +173,7 @@ public class ActiveRuleIndex extends BaseIndex getByKeys(Collection keys) { - return index.getByKeys(keys); + throw new UnsupportedOperationException("Please use RuleDao"); } + @Deprecated public Rule getNonNullByKey(RuleKey key) { - Rule rule = index.getNullableByKey(key); - if (rule == null) { - throw new NotFoundException("Rule not found: " + key); - } - return rule; + throw new UnsupportedOperationException("Please use RuleDao"); } public RuleQuery newRuleQuery() { @@ -85,7 +83,7 @@ public class RuleService { */ public Set listTags() { /** using combined ALL_TAGS field of ES until ES update that has multiTerms aggregation */ - return index.terms(RuleNormalizer.RuleField.ALL_TAGS.field()); + return index.terms(RuleIndexDefinition.FIELD_RULE_ALL_TAGS); } /** @@ -93,7 +91,7 @@ public class RuleService { */ public Set listTags(@Nullable String query, int size) { /** using combined ALL_TAGS field of ES until ES update that has multiTerms aggregation */ - return index.terms(RuleNormalizer.RuleField.ALL_TAGS.field(), query, size); + return index.terms(RuleIndexDefinition.FIELD_RULE_ALL_TAGS, query, size); } public RuleKey create(NewRule newRule) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleDoc.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleDoc.java index 1a9d8c61098..7dbaa7f185a 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleDoc.java @@ -19,9 +19,11 @@ */ package org.sonar.server.rule.index; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Predicate; import com.google.common.collect.Iterables; import java.util.ArrayList; +import java.util.Collection; import java.util.Date; import java.util.List; import java.util.Map; @@ -49,11 +51,6 @@ public class RuleDoc extends BaseDoc implements Rule { super(fields); } - @Override - public RuleKey key() { - return RuleKey.parse(this.getField(RuleNormalizer.RuleField.KEY.field())); - } - /** * @deprecated Only use for sqale backward compat. Use key() instead. */ @@ -62,34 +59,87 @@ public class RuleDoc extends BaseDoc implements Rule { return getField(RuleNormalizer.RuleField.ID.field()); } - /** - * Alias for backward-compatibility with SQALE - */ - public RuleKey ruleKey() { - return key(); + @Override + public RuleKey key() { + return RuleKey.parse(this.getField(RuleIndexDefinition.FIELD_RULE_KEY)); + } + + public RuleDoc setKey(@Nullable String s) { + setField(RuleIndexDefinition.FIELD_RULE_KEY, s); + return this; + } + + @VisibleForTesting + List keyAsList() { + return (List) getField(RuleIndexDefinition.FIELD_RULE_KEY_AS_LIST); + } + + public RuleDoc setKeyAsList(@Nullable List s) { + setField(RuleIndexDefinition.FIELD_RULE_KEY_AS_LIST, s); + return this; + } + + @CheckForNull + public String ruleKey() { + return getNullableField(RuleIndexDefinition.FIELD_RULE_RULE_KEY); + } + + public RuleDoc setRuleKey(@Nullable String s) { + setField(RuleIndexDefinition.FIELD_RULE_RULE_KEY, s); + return this; + } + + @CheckForNull + public String repository() { + return getNullableField(RuleIndexDefinition.FIELD_RULE_REPOSITORY); + } + + public RuleDoc setRepository(@Nullable String s) { + setField(RuleIndexDefinition.FIELD_RULE_REPOSITORY, s); + return this; } @Override @CheckForNull public String internalKey() { - return getNullableField(RuleNormalizer.RuleField.INTERNAL_KEY.field()); + return getNullableField(RuleIndexDefinition.FIELD_RULE_INTERNAL_KEY); + } + + public RuleDoc setInternalKey(@Nullable String s) { + setField(RuleIndexDefinition.FIELD_RULE_INTERNAL_KEY, s); + return this; } @Override @CheckForNull public String language() { - return getNullableField(RuleNormalizer.RuleField.LANGUAGE.field()); + return getNullableField(RuleIndexDefinition.FIELD_RULE_LANGUAGE); + } + + public RuleDoc setLanguage(@Nullable String s) { + setField(RuleIndexDefinition.FIELD_RULE_LANGUAGE, s); + return this; } @Override public String name() { - return getField(RuleNormalizer.RuleField.NAME.field()); + return getField(RuleIndexDefinition.FIELD_RULE_NAME); + } + + public RuleDoc setName(@Nullable String s) { + setField(RuleIndexDefinition.FIELD_RULE_NAME, s); + return this; } @Override @CheckForNull public String htmlDescription() { - return getNullableField(RuleNormalizer.RuleField.HTML_DESCRIPTION.field()); + return getNullableField(RuleIndexDefinition.FIELD_RULE_HTML_DESCRIPTION); + } + + public RuleDoc setHtmlDescription(@Nullable String s) { + setField(RuleIndexDefinition.FIELD_RULE_HTML_DESCRIPTION, s); + return this; } @Override @@ -107,18 +157,33 @@ public class RuleDoc extends BaseDoc implements Rule { @Override @CheckForNull public String severity() { - return (String) getNullableField(RuleNormalizer.RuleField.SEVERITY.field()); + return (String) getNullableField(RuleIndexDefinition.FIELD_RULE_SEVERITY); + } + + public RuleDoc setSeverity(@Nullable String s) { + setField(RuleIndexDefinition.FIELD_RULE_SEVERITY, s); + return this; } @Override @CheckForNull public RuleStatus status() { - return RuleStatus.valueOf((String) getField(RuleNormalizer.RuleField.STATUS.field())); + return RuleStatus.valueOf((String) getField(RuleIndexDefinition.FIELD_RULE_STATUS)); + } + + public RuleDoc setStatus(@Nullable String s) { + setField(RuleIndexDefinition.FIELD_RULE_STATUS, s); + return this; } @Override public boolean isTemplate() { - return (Boolean) getField(RuleNormalizer.RuleField.IS_TEMPLATE.field()); + return (Boolean) getField(RuleIndexDefinition.FIELD_RULE_IS_TEMPLATE); + } + + public RuleDoc setIsTemplate(@Nullable Boolean b) { + setField(RuleIndexDefinition.FIELD_RULE_IS_TEMPLATE, b); + return this; } @Override @@ -128,6 +193,11 @@ public class RuleDoc extends BaseDoc implements Rule { return templateKey != null ? RuleKey.parse(templateKey) : null; } + public RuleDoc setTemplateKey(@Nullable String s) { + setField(RuleIndexDefinition.FIELD_RULE_TEMPLATE_KEY, s); + return this; + } + @Override public List tags() { return (List) getField(RuleNormalizer.RuleField.TAGS.field()); @@ -138,6 +208,15 @@ public class RuleDoc extends BaseDoc implements Rule { return (List) getField(RuleNormalizer.RuleField.SYSTEM_TAGS.field()); } + public Collection allTags() { + return (Collection) getField(RuleIndexDefinition.FIELD_RULE_ALL_TAGS); + } + + public RuleDoc setAllTags(@Nullable Collection l) { + setField(RuleIndexDefinition.FIELD_RULE_ALL_TAGS, l); + return this; + } + @Override public List params() { List params = new ArrayList<>(); @@ -267,11 +346,31 @@ public class RuleDoc extends BaseDoc implements Rule { return IndexUtils.parseDateTime((String) getNullableField(RuleNormalizer.RuleField.CREATED_AT.field())); } + @CheckForNull + public Long createdAtAsLong() { + return (Long) getField(RuleIndexDefinition.FIELD_RULE_CREATED_AT); + } + + public RuleDoc setCreatedAt(@Nullable Long l) { + setField(RuleIndexDefinition.FIELD_RULE_CREATED_AT, l); + return this; + } + @Override public Date updatedAt() { return IndexUtils.parseDateTime((String) getNullableField(RuleNormalizer.RuleField.UPDATED_AT.field())); } + @CheckForNull + public Long updatedAtAtAsLong() { + return (Long) getField(RuleIndexDefinition.FIELD_RULE_UPDATED_AT); + } + + public RuleDoc setUpdatedAt(@Nullable Long l) { + setField(RuleIndexDefinition.FIELD_RULE_UPDATED_AT, l); + return this; + } + @Override public boolean isManual() { return getField(RuleNormalizer.RuleField.REPOSITORY.field()).equals(MANUAL_REPOSITORY); diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java index 4589a8fec2a..fdadd3d1f62 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java @@ -151,7 +151,7 @@ public class RuleIndex extends BaseIndex { /* integrate Query Sort */ String queryText = query.getQueryText(); if (query.getSortField() != null) { - FieldSortBuilder sort = SortBuilders.fieldSort(query.getSortField().sortField()); + FieldSortBuilder sort = SortBuilders.fieldSort(query.getSortField()); if (query.isAscendingSort()) { sort.order(SortOrder.ASC); } else { diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex2.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex2.java new file mode 100644 index 00000000000..2fbb0fcef1d --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex2.java @@ -0,0 +1,493 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.rule.index; + +import com.google.common.base.Function; +import com.google.common.base.Joiner; +import com.google.common.base.Predicate; +import com.google.common.collect.Collections2; +import com.google.common.collect.ImmutableList; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.BoolFilterBuilder; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.FilterBuilder; +import org.elasticsearch.index.query.FilterBuilders; +import org.elasticsearch.index.query.HasParentFilterBuilder; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.SimpleQueryStringBuilder; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.sort.SortOrder; +import org.sonar.api.rule.RuleKey; +import org.sonar.api.rule.RuleStatus; +import org.sonar.api.rule.Severity; +import org.sonar.server.es.BaseIndex; +import org.sonar.server.es.EsClient; +import org.sonar.server.es.SearchIdResult; +import org.sonar.server.es.SearchOptions; +import org.sonar.server.qualityprofile.index.ActiveRuleNormalizer; +import org.sonar.server.search.IndexDefinition; +import org.sonar.server.search.IndexField; +import org.sonar.server.search.StickyFacetBuilder; + +import static org.sonar.server.es.EsUtils.SCROLL_TIME_IN_MINUTES; +import static org.sonar.server.es.EsUtils.scrollIds; + +/** + * The unique entry-point to interact with Elasticsearch index "rules". + * All the requests are listed here. + */ +public class RuleIndex2 extends BaseIndex { + + public static final String FACET_LANGUAGES = "languages"; + public static final String FACET_TAGS = "tags"; + public static final String FACET_REPOSITORIES = "repositories"; + public static final String FACET_SEVERITIES = "severities"; + public static final String FACET_ACTIVE_SEVERITIES = "active_severities"; + public static final String FACET_STATUSES = "statuses"; + public static final String FACET_OLD_DEFAULT = "true"; + + public static final List ALL_STATUSES_EXCEPT_REMOVED = ImmutableList.copyOf( + Collections2.filter( + Collections2.transform( + Arrays.asList(RuleStatus.values()), + new Function() { + @Override + public String apply(@Nonnull RuleStatus input) { + return input.toString(); + } + }), + new Predicate() { + @Override + public boolean apply(@Nonnull String input) { + return !RuleStatus.REMOVED.toString().equals(input); + } + })); + + public RuleIndex2(EsClient client) { + super(client); + } + + public SearchIdResult search(RuleQuery query, SearchOptions options) { + SearchRequestBuilder esSearch = getClient() + .prepareSearch(RuleIndexDefinition.INDEX) + .setTypes(RuleIndexDefinition.TYPE_RULE); + + QueryBuilder qb = buildQuery(query); + Map filters = buildFilters(query); + + if (!options.getFacets().isEmpty()) { + for (AggregationBuilder aggregation : getFacets(query, options, qb, filters).values()) { + esSearch.addAggregation(aggregation); + } + } + + setSorting(query, esSearch); + setPagination(options, esSearch); + + BoolFilterBuilder fb = FilterBuilders.boolFilter(); + for (FilterBuilder filterBuilder : filters.values()) { + fb.must(filterBuilder); + } + + esSearch.setQuery(QueryBuilders.filteredQuery(qb, fb)); + return new SearchIdResult<>(esSearch.get(), ToRuleKey.INSTANCE); + } + + /** + * Return all keys matching the search query, without pagination nor facets + */ + public Iterator searchAll(RuleQuery query) { + SearchRequestBuilder esSearch = getClient() + .prepareSearch(RuleIndexDefinition.INDEX) + .setTypes(RuleIndexDefinition.TYPE_RULE) + .setSearchType(SearchType.SCAN) + .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)); + + QueryBuilder qb = buildQuery(query); + Map filters = buildFilters(query); + setSorting(query, esSearch); + + BoolFilterBuilder fb = FilterBuilders.boolFilter(); + for (FilterBuilder filterBuilder : filters.values()) { + fb.must(filterBuilder); + } + + esSearch.setQuery(QueryBuilders.filteredQuery(qb, fb)); + SearchResponse response = esSearch.get(); + return scrollIds(getClient(), response.getScrollId(), ToRuleKey.INSTANCE); + } + + /* Build main query (search based) */ + private QueryBuilder buildQuery(RuleQuery query) { + + // No contextual query case + String queryText = query.getQueryText(); + if (queryText == null || queryText.isEmpty()) { + return QueryBuilders.matchAllQuery(); + } + + // Build RuleBased contextual query + BoolQueryBuilder qb = QueryBuilders.boolQuery(); + String queryString = query.getQueryText(); + + // Human readable type of querying + qb.should(QueryBuilders.simpleQueryStringQuery(query.getQueryText()) + .field(RuleIndexDefinition.FIELD_RULE_NAME + "." + BaseIndex.SEARCH_WORDS_SUFFIX, 20f) + .field(RuleIndexDefinition.FIELD_RULE_HTML_DESCRIPTION + "." + BaseIndex.SEARCH_WORDS_SUFFIX, 3f) + .defaultOperator(SimpleQueryStringBuilder.Operator.AND) + ).boost(20f); + + // Match and partial Match queries + qb.should(this.termQuery(RuleIndexDefinition.FIELD_RULE_KEY, queryString, 15f)); + qb.should(this.termQuery(RuleIndexDefinition.FIELD_RULE_KEY_AS_LIST, queryString, 35f)); + qb.should(this.termQuery(RuleIndexDefinition.FIELD_RULE_LANGUAGE, queryString, 3f)); + qb.should(this.termQuery(RuleIndexDefinition.FIELD_RULE_ALL_TAGS, queryString, 10f)); + qb.should(this.termAnyQuery(RuleIndexDefinition.FIELD_RULE_ALL_TAGS, queryString, 1f)); + + return qb; + } + + private QueryBuilder termQuery(String field, String query, float boost) { + return QueryBuilders.multiMatchQuery(query, + field, field + "." + IndexField.SEARCH_PARTIAL_SUFFIX) + .operator(MatchQueryBuilder.Operator.AND) + .boost(boost); + } + + private QueryBuilder termAnyQuery(String field, String query, float boost) { + return QueryBuilders.multiMatchQuery(query, + field, field + "." + IndexField.SEARCH_PARTIAL_SUFFIX) + .operator(MatchQueryBuilder.Operator.OR) + .boost(boost); + } + + /* Build main filter (match based) */ + private Map buildFilters(RuleQuery query) { + + Map filters = new HashMap<>(); + + /* Add enforced filter on rules that are REMOVED */ + filters.put(RuleIndexDefinition.FIELD_RULE_STATUS, + FilterBuilders.boolFilter().mustNot( + FilterBuilders.termFilter(RuleIndexDefinition.FIELD_RULE_STATUS, + RuleStatus.REMOVED.toString()))); + + if (!StringUtils.isEmpty(query.getInternalKey())) { + filters.put(RuleIndexDefinition.FIELD_RULE_INTERNAL_KEY, + FilterBuilders.termFilter(RuleIndexDefinition.FIELD_RULE_INTERNAL_KEY, query.getInternalKey())); + } + + if (!StringUtils.isEmpty(query.getRuleKey())) { + filters.put(RuleIndexDefinition.FIELD_RULE_RULE_KEY, + FilterBuilders.termFilter(RuleIndexDefinition.FIELD_RULE_RULE_KEY, query.getRuleKey())); + } + + if (!CollectionUtils.isEmpty(query.getLanguages())) { + filters.put(RuleIndexDefinition.FIELD_RULE_LANGUAGE, + FilterBuilders.termsFilter(RuleIndexDefinition.FIELD_RULE_LANGUAGE, query.getLanguages())); + } + + if (!CollectionUtils.isEmpty(query.getRepositories())) { + filters.put(RuleIndexDefinition.FIELD_RULE_REPOSITORY, + FilterBuilders.termsFilter(RuleIndexDefinition.FIELD_RULE_REPOSITORY, query.getRepositories())); + } + + if (!CollectionUtils.isEmpty(query.getSeverities())) { + filters.put(RuleIndexDefinition.FIELD_RULE_SEVERITY, + FilterBuilders.termsFilter(RuleIndexDefinition.FIELD_RULE_SEVERITY, query.getSeverities())); + } + + if (!StringUtils.isEmpty(query.getKey())) { + filters.put(RuleIndexDefinition.FIELD_RULE_KEY, + FilterBuilders.termFilter(RuleIndexDefinition.FIELD_RULE_KEY, query.getKey())); + } + + if (!CollectionUtils.isEmpty(query.getTags())) { + filters.put(RuleIndexDefinition.FIELD_RULE_ALL_TAGS, + FilterBuilders.termsFilter(RuleIndexDefinition.FIELD_RULE_ALL_TAGS, query.getTags())); + } + + if (query.getAvailableSinceLong() != null) { + filters.put("availableSince", FilterBuilders.rangeFilter(RuleIndexDefinition.FIELD_RULE_CREATED_AT) + .gte(query.getAvailableSinceLong())); + } + + Collection statusValues = query.getStatuses(); + if (statusValues != null && !statusValues.isEmpty()) { + Collection stringStatus = new ArrayList<>(); + for (RuleStatus status : statusValues) { + stringStatus.add(status.name()); + } + filters.put(RuleIndexDefinition.FIELD_RULE_STATUS, + FilterBuilders.termsFilter(RuleIndexDefinition.FIELD_RULE_STATUS, stringStatus)); + } + + Boolean isTemplate = query.isTemplate(); + if (isTemplate != null) { + filters.put(RuleIndexDefinition.FIELD_RULE_IS_TEMPLATE, + FilterBuilders.termFilter(RuleIndexDefinition.FIELD_RULE_IS_TEMPLATE, Boolean.toString(isTemplate))); + } + + String template = query.templateKey(); + if (template != null) { + filters.put(RuleIndexDefinition.FIELD_RULE_TEMPLATE_KEY, + FilterBuilders.termFilter(RuleIndexDefinition.FIELD_RULE_TEMPLATE_KEY, template)); + } + + // ActiveRule Filter (profile and inheritance) + BoolFilterBuilder childrenFilter = FilterBuilders.boolFilter(); + this.addTermFilter(childrenFilter, ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field(), query.getQProfileKey()); + this.addTermFilter(childrenFilter, ActiveRuleNormalizer.ActiveRuleField.INHERITANCE.field(), query.getInheritance()); + this.addTermFilter(childrenFilter, ActiveRuleNormalizer.ActiveRuleField.SEVERITY.field(), query.getActiveSeverities()); + + // ChildQuery + FilterBuilder childQuery; + if (childrenFilter.hasClauses()) { + childQuery = childrenFilter; + } else { + childQuery = FilterBuilders.matchAllFilter(); + } + + /** Implementation of activation query */ + if (Boolean.TRUE.equals(query.getActivation())) { + filters.put("activation", + FilterBuilders.hasChildFilter(IndexDefinition.ACTIVE_RULE.getIndexType(), + childQuery)); + } else if (Boolean.FALSE.equals(query.getActivation())) { + filters.put("activation", + FilterBuilders.boolFilter().mustNot( + FilterBuilders.hasChildFilter(IndexDefinition.ACTIVE_RULE.getIndexType(), + childQuery))); + } + + return filters; + } + + private BoolFilterBuilder addTermFilter(BoolFilterBuilder filter, String field, @Nullable Collection values) { + if (values != null && !values.isEmpty()) { + BoolFilterBuilder valuesFilter = FilterBuilders.boolFilter(); + for (String value : values) { + FilterBuilder valueFilter = FilterBuilders.termFilter(field, value); + valuesFilter.should(valueFilter); + } + filter.must(valuesFilter); + } + return filter; + } + + private BoolFilterBuilder addTermFilter(BoolFilterBuilder filter, String field, @Nullable String value) { + if (value != null && !value.isEmpty()) { + filter.must(FilterBuilders.termFilter(field, value)); + } + return filter; + } + + private Map getFacets(RuleQuery query, SearchOptions options, QueryBuilder queryBuilder, Map filters) { + Map aggregations = new HashMap<>(); + StickyFacetBuilder stickyFacetBuilder = stickyFacetBuilder(queryBuilder, filters); + + addDefaultFacets(query, options, aggregations, stickyFacetBuilder); + + addStatusFacetIfNeeded(options, aggregations, stickyFacetBuilder); + + if (options.getFacets().contains(FACET_SEVERITIES)) { + aggregations.put(FACET_SEVERITIES, + stickyFacetBuilder.buildStickyFacet(RuleIndexDefinition.FIELD_RULE_SEVERITY, FACET_SEVERITIES, Severity.ALL.toArray())); + } + + addActiveSeverityFacetIfNeeded(query, options, aggregations, stickyFacetBuilder); + return aggregations; + } + + private void addDefaultFacets(RuleQuery query, SearchOptions options, Map aggregations, StickyFacetBuilder stickyFacetBuilder) { + if (options.getFacets().contains(FACET_LANGUAGES) || options.getFacets().contains(FACET_OLD_DEFAULT)) { + Collection languages = query.getLanguages(); + aggregations.put(FACET_LANGUAGES, + stickyFacetBuilder.buildStickyFacet(RuleIndexDefinition.FIELD_RULE_LANGUAGE, FACET_LANGUAGES, + languages == null ? new String[0] : languages.toArray())); + } + if (options.getFacets().contains(FACET_TAGS) || options.getFacets().contains(FACET_OLD_DEFAULT)) { + Collection tags = query.getTags(); + aggregations.put(FACET_TAGS, + stickyFacetBuilder.buildStickyFacet(RuleIndexDefinition.FIELD_RULE_ALL_TAGS, FACET_TAGS, + tags == null ? new String[0] : tags.toArray())); + } + if (options.getFacets().contains("repositories") || options.getFacets().contains(FACET_OLD_DEFAULT)) { + Collection repositories = query.getRepositories(); + aggregations.put(FACET_REPOSITORIES, + stickyFacetBuilder.buildStickyFacet(RuleIndexDefinition.FIELD_RULE_REPOSITORY, FACET_REPOSITORIES, + repositories == null ? new String[0] : repositories.toArray())); + } + } + + private void addStatusFacetIfNeeded(SearchOptions options, Map aggregations, StickyFacetBuilder stickyFacetBuilder) { + if (options.getFacets().contains(FACET_STATUSES)) { + BoolFilterBuilder facetFilter = stickyFacetBuilder.getStickyFacetFilter(RuleIndexDefinition.FIELD_RULE_STATUS); + AggregationBuilder statuses = AggregationBuilders.filter(FACET_STATUSES + "_filter") + .filter(facetFilter) + .subAggregation( + AggregationBuilders + .terms(FACET_STATUSES) + .field(RuleIndexDefinition.FIELD_RULE_STATUS) + .include(Joiner.on('|').join(ALL_STATUSES_EXCEPT_REMOVED)) + .exclude(RuleStatus.REMOVED.toString()) + .size(ALL_STATUSES_EXCEPT_REMOVED.size())); + + aggregations.put(FACET_STATUSES, AggregationBuilders.global(FACET_STATUSES).subAggregation(statuses)); + } + } + + private void addActiveSeverityFacetIfNeeded(RuleQuery query, SearchOptions options, Map aggregations, StickyFacetBuilder stickyFacetBuilder) { + if (options.getFacets().contains(FACET_ACTIVE_SEVERITIES)) { + // We are building a children aggregation on active rules + // so the rule filter has to be used as parent filter for active rules + // from which we remove filters that concern active rules ("activation") + HasParentFilterBuilder ruleFilter = FilterBuilders.hasParentFilter( + IndexDefinition.RULE.getIndexType(), + stickyFacetBuilder.getStickyFacetFilter("activation")); + + // Rebuilding the active rule filter without severities + BoolFilterBuilder childrenFilter = FilterBuilders.boolFilter(); + this.addTermFilter(childrenFilter, ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field(), query.getQProfileKey()); + this.addTermFilter(childrenFilter, ActiveRuleNormalizer.ActiveRuleField.INHERITANCE.field(), query.getInheritance()); + FilterBuilder activeRuleFilter; + if (childrenFilter.hasClauses()) { + activeRuleFilter = childrenFilter.must(ruleFilter); + } else { + activeRuleFilter = ruleFilter; + } + + AggregationBuilder activeSeverities = AggregationBuilders.children(FACET_ACTIVE_SEVERITIES + "_children") + .childType(IndexDefinition.ACTIVE_RULE.getIndexType()) + .subAggregation(AggregationBuilders.filter(FACET_ACTIVE_SEVERITIES + "_filter") + .filter(activeRuleFilter) + .subAggregation( + AggregationBuilders + .terms(FACET_ACTIVE_SEVERITIES) + .field(ActiveRuleNormalizer.ActiveRuleField.SEVERITY.field()) + .include(Joiner.on('|').join(Severity.ALL)) + .size(Severity.ALL.size()))); + + aggregations.put(FACET_ACTIVE_SEVERITIES, AggregationBuilders.global(FACET_ACTIVE_SEVERITIES).subAggregation(activeSeverities)); + } + } + + private StickyFacetBuilder stickyFacetBuilder(QueryBuilder query, Map filters) { + return new StickyFacetBuilder(query, filters); + } + + private void setSorting(RuleQuery query, SearchRequestBuilder esSearch) { + /* integrate Query Sort */ + String queryText = query.getQueryText(); + if (query.getSortField() != null) { + FieldSortBuilder sort = SortBuilders.fieldSort(appendSortSuffixIfNeeded(query.getSortField())); + if (query.isAscendingSort()) { + sort.order(SortOrder.ASC); + } else { + sort.order(SortOrder.DESC); + } + esSearch.addSort(sort); + } else if (queryText != null && !queryText.isEmpty()) { + esSearch.addSort(SortBuilders.scoreSort()); + } else { + esSearch.addSort(appendSortSuffixIfNeeded(RuleIndexDefinition.FIELD_RULE_UPDATED_AT), SortOrder.DESC); + // deterministic sort when exactly the same updated_at (same millisecond) + esSearch.addSort(appendSortSuffixIfNeeded(RuleIndexDefinition.FIELD_RULE_KEY), SortOrder.ASC); + } + } + + public static String appendSortSuffixIfNeeded(String field) { + return field + + ((field.equals(RuleIndexDefinition.FIELD_RULE_NAME) + || field.equals(RuleIndexDefinition.FIELD_RULE_KEY)) + ? "." + BaseIndex.SORT_SUFFIX + : ""); + } + + private void setPagination(SearchOptions options, SearchRequestBuilder esSearch) { + esSearch.setFrom(options.getOffset()); + esSearch.setSize(options.getLimit()); + } + + public Set terms(String fields) { + return terms(fields, null, Integer.MAX_VALUE); + } + + public Set terms(String fields, @Nullable String query, int size) { + Set tags = new HashSet<>(); + String key = "_ref"; + + TermsBuilder terms = AggregationBuilders.terms(key) + .field(fields) + .size(size) + .minDocCount(1); + if (query != null) { + terms.include(".*" + query + ".*"); + } + SearchRequestBuilder request = this.getClient() + .prepareSearch(RuleIndexDefinition.INDEX) + .setQuery(QueryBuilders.matchAllQuery()) + .addAggregation(terms); + + SearchResponse esResponse = request.get(); + + Terms aggregation = esResponse.getAggregations().get(key); + + if (aggregation != null) { + for (Terms.Bucket value : aggregation.getBuckets()) { + tags.add(value.getKey()); + } + } + return tags; + } + + private enum ToRuleKey implements Function { + INSTANCE; + + @Override + public RuleKey apply(@Nonnull String input) { + return RuleKey.parse(input); + } + } + +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java new file mode 100644 index 00000000000..4096edfcece --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java @@ -0,0 +1,100 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.rule.index; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import java.util.Set; +import org.sonar.api.config.Settings; +import org.sonar.server.es.IndexDefinition; +import org.sonar.server.es.NewIndex; + +/** + * Definition of ES index "rules", including settings and fields. + */ +public class RuleIndexDefinition implements IndexDefinition { + + public static final String INDEX = "rules"; + public static final String TYPE_RULE = "rule"; + + public static final String FIELD_RULE_KEY = "key"; + // TODO find at what this field is useful ? + public static final String FIELD_RULE_KEY_AS_LIST = "_key"; + public static final String FIELD_RULE_REPOSITORY = "repo"; + public static final String FIELD_RULE_RULE_KEY = "ruleKey"; + public static final String FIELD_RULE_INTERNAL_KEY = "internalKey"; + public static final String FIELD_RULE_NAME = "name"; + public static final String FIELD_RULE_HTML_DESCRIPTION = "htmlDesc"; + public static final String FIELD_RULE_SEVERITY = "severity"; + public static final String FIELD_RULE_STATUS = "status"; + public static final String FIELD_RULE_LANGUAGE = "lang"; + public static final String FIELD_RULE_IS_TEMPLATE = "isTemplate"; + public static final String FIELD_RULE_TEMPLATE_KEY = "templateKey"; + public static final String FIELD_RULE_ALL_TAGS = "allTags"; + public static final String FIELD_RULE_CREATED_AT = "createdAt"; + public static final String FIELD_RULE_UPDATED_AT = "updatedAt"; + + public static final Set SORT_FIELDS = ImmutableSet.of( + RuleIndexDefinition.FIELD_RULE_NAME, + RuleIndexDefinition.FIELD_RULE_UPDATED_AT, + RuleIndexDefinition.FIELD_RULE_CREATED_AT, + RuleIndexDefinition.FIELD_RULE_KEY + ); + + private final Settings settings; + + public RuleIndexDefinition(Settings settings) { + this.settings = settings; + } + + @Override + public void define(IndexDefinitionContext context) { + NewIndex index = context.create(INDEX); + + index.refreshHandledByIndexer(); + index.setShards(settings); + + // Rule type + NewIndex.NewIndexType ruleMapping = index.createType(TYPE_RULE); + ruleMapping.setAttribute("_id", ImmutableMap.of("path", FIELD_RULE_KEY)); + ruleMapping.setAttribute("_routing", ImmutableMap.of("required", true, "path", RuleIndexDefinition.FIELD_RULE_REPOSITORY)); + ruleMapping.setEnableSource(false); + + ruleMapping.stringFieldBuilder(FIELD_RULE_KEY).enableSorting().enableGramSearch().build(); + ruleMapping.stringFieldBuilder(FIELD_RULE_KEY_AS_LIST).enableGramSearch().build(); + ruleMapping.stringFieldBuilder(FIELD_RULE_RULE_KEY).disableSearch().docValues().build(); + ruleMapping.stringFieldBuilder(FIELD_RULE_REPOSITORY).docValues().build(); + ruleMapping.stringFieldBuilder(FIELD_RULE_INTERNAL_KEY).disableSearch().docValues().build(); + + ruleMapping.stringFieldBuilder(FIELD_RULE_NAME).enableSorting().enableWordSearch().build(); + ruleMapping.stringFieldBuilder(FIELD_RULE_HTML_DESCRIPTION).enableWordSearch().build(); + ruleMapping.stringFieldBuilder(FIELD_RULE_SEVERITY).docValues().build(); + ruleMapping.stringFieldBuilder(FIELD_RULE_STATUS).docValues().build(); + ruleMapping.stringFieldBuilder(FIELD_RULE_LANGUAGE).enableGramSearch().build(); + + ruleMapping.createBooleanField(FIELD_RULE_IS_TEMPLATE); + ruleMapping.stringFieldBuilder(FIELD_RULE_TEMPLATE_KEY).docValues().build(); + + ruleMapping.stringFieldBuilder(FIELD_RULE_ALL_TAGS).enableGramSearch().build(); + + ruleMapping.createLongField(FIELD_RULE_CREATED_AT); + ruleMapping.createLongField(FIELD_RULE_UPDATED_AT); + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java new file mode 100644 index 00000000000..7354a5b6e8f --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java @@ -0,0 +1,101 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.rule.index; + +import java.util.Iterator; +import org.elasticsearch.action.index.IndexRequest; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.server.es.BaseIndexer; +import org.sonar.server.es.BulkIndexer; +import org.sonar.server.es.EsClient; + +import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_UPDATED_AT; +import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX; +import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE; + +public class RuleIndexer extends BaseIndexer { + + private final DbClient dbClient; + + public RuleIndexer(DbClient dbClient, EsClient esClient) { + super(esClient, 300, INDEX, TYPE_RULE, FIELD_RULE_UPDATED_AT); + this.dbClient = dbClient; + } + + @Override + protected long doIndex(long lastUpdatedAt) { + return doIndex(createBulkIndexer(false), lastUpdatedAt); + } + + public void index(Iterator rules) { + doIndex(createBulkIndexer(false), rules); + } + + private long doIndex(BulkIndexer bulk, long lastUpdatedAt) { + DbSession dbSession = dbClient.openSession(false); + long maxDate; + try { + RuleResultSetIterator rowIt = RuleResultSetIterator.create(dbClient, dbSession, lastUpdatedAt); + maxDate = doIndex(bulk, rowIt); + rowIt.close(); + return maxDate; + } finally { + dbSession.close(); + } + } + + private long doIndex(BulkIndexer bulk, Iterator rules) { + bulk.start(); + long maxDate = 0L; + while (rules.hasNext()) { + RuleDoc rule = rules.next(); + // TODO when active rule is not more DAO v2, restore deleting of REMOVED rules and also remove active rules linked to this rule +// if (rule.status() == RuleStatus.REMOVED) { +// bulk.add(newDeleteRequest(rule)); +// } else { +// } + bulk.add(newUpsertRequest(rule)); + + // it's more efficient to sort programmatically than in SQL on some databases (MySQL for instance) + maxDate = Math.max(maxDate, rule.updatedAtAtAsLong()); + } + bulk.stop(); + return maxDate; + } + + private BulkIndexer createBulkIndexer(boolean large) { + BulkIndexer bulk = new BulkIndexer(esClient, INDEX); + bulk.setLarge(large); + return bulk; + } + + private IndexRequest newUpsertRequest(RuleDoc rule) { + return new IndexRequest(INDEX, TYPE_RULE, rule.key().toString()) + .routing(rule.repository()) + .source(rule.getFields()); + } + +// private DeleteRequest newDeleteRequest(RuleDoc rule) { +// return new DeleteRequest(INDEX, TYPE_RULE, rule.key().toString()) +// .routing(rule.repository()); +// } + +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleQuery.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleQuery.java index 45c1d847f4f..2fb6837665f 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleQuery.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleQuery.java @@ -20,15 +20,12 @@ package org.sonar.server.rule.index; import com.google.common.base.Preconditions; -import org.sonar.api.rule.RuleStatus; -import org.sonar.api.rule.Severity; -import org.sonar.server.search.IndexField; - -import javax.annotation.CheckForNull; -import javax.annotation.Nullable; - import java.util.Collection; import java.util.Date; +import javax.annotation.CheckForNull; +import javax.annotation.Nullable; +import org.sonar.api.rule.RuleStatus; +import org.sonar.api.rule.Severity; public class RuleQuery { @@ -45,8 +42,8 @@ public class RuleQuery { private Collection activeSeverities; private String templateKey; private Boolean isTemplate; - private Date availableSince; - private IndexField sortField; + private Long availableSince; + private String sortField; private boolean ascendingSort = true; private String internalKey; private String ruleKey; @@ -203,15 +200,15 @@ public class RuleQuery { return this; } - public IndexField getSortField() { + public String getSortField() { return this.sortField; } - public RuleQuery setSortField(@Nullable IndexField sf) { - if (sf != null && !sf.isSortable()) { - throw new IllegalStateException(String.format("Field '%s' is not sortable", sf.field())); + public RuleQuery setSortField(@Nullable String field) { + if (field != null && !RuleIndexDefinition.SORT_FIELDS.contains(field)) { + throw new IllegalStateException(String.format("Field '%s' is not sortable", field)); } - this.sortField = sf; + this.sortField = field; return this; } @@ -224,12 +221,23 @@ public class RuleQuery { return this; } + @Deprecated public RuleQuery setAvailableSince(@Nullable Date d) { - this.availableSince = d; + this.availableSince = d.getTime(); return this; } + @Deprecated public Date getAvailableSince() { + return new Date(this.availableSince); + } + + public RuleQuery setAvailableSince(@Nullable Long l) { + this.availableSince = l; + return this; + } + + public Long getAvailableSinceLong() { return this.availableSince; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleResultSetIterator.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleResultSetIterator.java new file mode 100644 index 00000000000..cf223abe9f5 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleResultSetIterator.java @@ -0,0 +1,141 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.rule.index; + +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Set; +import javax.annotation.Nullable; +import org.apache.commons.lang.StringUtils; +import org.sonar.api.rule.RuleKey; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.ResultSetIterator; +import org.sonar.db.rule.RuleDto; +import org.sonar.db.rule.SeverityUtil; +import org.sonar.markdown.Markdown; + +/** + * Scrolls over table RULES and reads documents to populate the rules index + */ +public class RuleResultSetIterator extends ResultSetIterator { + + private static final String[] FIELDS = { + // column 1 + "r.plugin_rule_key", + "r.plugin_name", + "r.name", + "r.description", + "r.description_format", + "r.priority", + "r.status", + "r.is_template", + "r.tags", + "r.system_tags", + + // column 11 + "t.plugin_rule_key", + "t.plugin_name", + "r.plugin_config_key", + "r.language", + "r.created_at_ms", + "r.updated_at_ms", + }; + + private static final String SQL_ALL = "SELECT " + StringUtils.join(FIELDS, ",") + " FROM rules r " + + "LEFT OUTER JOIN rules t ON t.id=r.template_id"; + + private static final String SQL_AFTER_DATE = SQL_ALL + " WHERE r.updated_at_ms>?"; + + private static final Splitter TAGS_SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings(); + + private RuleResultSetIterator(PreparedStatement stmt) throws SQLException { + super(stmt); + } + + static RuleResultSetIterator create(DbClient dbClient, DbSession session, long afterDate) { + try { + String sql = afterDate > 0L ? SQL_AFTER_DATE : SQL_ALL; + PreparedStatement stmt = dbClient.getMyBatis().newScrollingSelectStatement(session, sql); + if (afterDate > 0L) { + stmt.setLong(1, afterDate); + } + return new RuleResultSetIterator(stmt); + } catch (SQLException e) { + throw new IllegalStateException("Fail to prepare SQL request to select all rules", e); + } + } + + @Override + protected RuleDoc read(ResultSet rs) throws SQLException { + RuleDoc doc = new RuleDoc(Maps.newHashMapWithExpectedSize(16)); + + String ruleKey = rs.getString(1); + String repositoryKey = rs.getString(2); + RuleKey key = RuleKey.of(repositoryKey, ruleKey); + + // all the fields must be present, even if value is null + doc.setKey(key.toString()); + doc.setKeyAsList(ImmutableList.of(repositoryKey, ruleKey)); + doc.setRuleKey(ruleKey); + doc.setRepository(repositoryKey); + doc.setName(rs.getString(3)); + + String description = rs.getString(4); + String descriptionFormat = rs.getString(5); + if (descriptionFormat != null) { + if (RuleDto.Format.HTML.equals(RuleDto.Format.valueOf(descriptionFormat))) { + doc.setHtmlDescription(description); + } else { + doc.setHtmlDescription(description == null ? null : Markdown.convertToHtml(description)); + } + } + + doc.setSeverity(SeverityUtil.getSeverityFromOrdinal(rs.getInt(6))); + doc.setStatus(rs.getString(7)); + doc.setIsTemplate(rs.getBoolean(8)); + doc.setAllTags(Sets.union(stringTagsToSet(rs.getString(9)), stringTagsToSet(rs.getString(10)))); + + String templateRuleKey = rs.getString(11); + String templateRepoKey = rs.getString(12); + if (templateRepoKey != null && templateRuleKey != null) { + doc.setTemplateKey(RuleKey.of(templateRepoKey, templateRuleKey).toString()); + } else { + doc.setTemplateKey(null); + } + + doc.setInternalKey(rs.getString(13)); + doc.setLanguage(rs.getString(14)); + doc.setCreatedAt(rs.getLong(15)); + doc.setUpdatedAt(rs.getLong(16)); + + return doc; + } + + private static Set stringTagsToSet(@Nullable String tags) { + return ImmutableSet.copyOf(TAGS_SPLITTER.split(tags == null ? "" : tags)); + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/ws/SearchAction.java b/server/sonar-server/src/main/java/org/sonar/server/rule/ws/SearchAction.java index b1f389781ab..97128fbbe3c 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/ws/SearchAction.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/ws/SearchAction.java @@ -55,8 +55,8 @@ import org.sonar.db.rule.RuleDto; import org.sonar.db.rule.RuleParamDto; import org.sonar.server.qualityprofile.ActiveRule; import org.sonar.server.rule.Rule; -import org.sonar.server.rule.index.RuleIndex; -import org.sonar.server.rule.index.RuleNormalizer; +import org.sonar.server.rule.index.RuleIndex2; +import org.sonar.server.rule.index.RuleIndexDefinition; import org.sonar.server.rule.index.RuleQuery; import org.sonar.server.search.FacetValue; import org.sonar.server.search.Facets; @@ -71,6 +71,15 @@ import static com.google.common.collect.FluentIterable.from; import static org.sonar.server.search.QueryContext.MAX_LIMIT; import static org.sonar.server.ws.WsUtils.writeProtobuf; +import static org.sonar.server.rule.index.RuleIndex2.ALL_STATUSES_EXCEPT_REMOVED; +import static org.sonar.server.rule.index.RuleIndex2.FACET_ACTIVE_SEVERITIES; +import static org.sonar.server.rule.index.RuleIndex2.FACET_LANGUAGES; +import static org.sonar.server.rule.index.RuleIndex2.FACET_OLD_DEFAULT; +import static org.sonar.server.rule.index.RuleIndex2.FACET_REPOSITORIES; +import static org.sonar.server.rule.index.RuleIndex2.FACET_SEVERITIES; +import static org.sonar.server.rule.index.RuleIndex2.FACET_STATUSES; +import static org.sonar.server.rule.index.RuleIndex2.FACET_TAGS; + /** * @since 4.4 */ @@ -97,12 +106,12 @@ public class SearchAction implements RulesWsAction { private final UserSession userSession; private final DbClient dbClient; - private final RuleIndex ruleIndex; + private final RuleIndex2 ruleIndex; private final ActiveRuleCompleter activeRuleCompleter; private final RuleMapping mapping; private final RuleMapper mapper; - public SearchAction(RuleIndex ruleIndex, ActiveRuleCompleter activeRuleCompleter, RuleMapping mapping, UserSession userSession, DbClient dbClient, RuleMapper mapper) { + public SearchAction(RuleIndex2 ruleIndex, ActiveRuleCompleter activeRuleCompleter, RuleMapping mapping, UserSession userSession, DbClient dbClient, RuleMapper mapper) { this.userSession = userSession; this.ruleIndex = ruleIndex; this.activeRuleCompleter = activeRuleCompleter; @@ -181,13 +190,14 @@ public class SearchAction implements RulesWsAction { @CheckForNull protected Collection possibleFacets() { return Arrays.asList( - RuleIndex.FACET_LANGUAGES, - RuleIndex.FACET_REPOSITORIES, - RuleIndex.FACET_TAGS, - RuleIndex.FACET_SEVERITIES, - RuleIndex.FACET_ACTIVE_SEVERITIES, - RuleIndex.FACET_STATUSES, - RuleIndex.FACET_OLD_DEFAULT); + FACET_LANGUAGES, + FACET_REPOSITORIES, + FACET_TAGS, + FACET_SEVERITIES, + FACET_ACTIVE_SEVERITIES, + FACET_STATUSES, + FACET_OLD_DEFAULT + ); } /** @@ -287,11 +297,9 @@ public class SearchAction implements RulesWsAction { action .createParam(Param.SORT) .setDescription("Sort field") - .setPossibleValues(RuleNormalizer.RuleField.NAME.field(), - RuleNormalizer.RuleField.UPDATED_AT.field(), - RuleNormalizer.RuleField.CREATED_AT.field(), - RuleNormalizer.RuleField.KEY.field()) - .setExampleValue(RuleNormalizer.RuleField.NAME.field()); + .setPossibleValues(RuleIndexDefinition.SORT_FIELDS) + .setExampleValue(RuleIndexDefinition.SORT_FIELDS.iterator().next()); + action .createParam(Param.ASCENDING) @@ -318,7 +326,7 @@ public class SearchAction implements RulesWsAction { String sortParam = request.param(Param.SORT); if (sortParam != null) { - query.setSortField(RuleNormalizer.RuleField.of(sortParam)); + query.setSortField(sortParam); query.setAscendingSort(request.mandatoryParamAsBoolean(Param.ASCENDING)); } return query; @@ -337,7 +345,7 @@ public class SearchAction implements RulesWsAction { .setLimit(context.getLimit()) .setOffset(context.getOffset()) .setScroll(context.isScroll()); - if (context.facets().contains(RuleIndex.FACET_OLD_DEFAULT)) { + if (context.facets().contains(RuleIndex2.FACET_OLD_DEFAULT)) { searchQueryContext.addFacets(DEFAULT_FACETS); } else { searchQueryContext.addFacets(context.facets()); @@ -424,12 +432,12 @@ public class SearchAction implements RulesWsAction { } protected void writeFacets(SearchResponse.Builder response, Request request, QueryContext context, SearchResult results) { - addMandatoryFacetValues(results, RuleIndex.FACET_LANGUAGES, request.paramAsStrings(PARAM_LANGUAGES)); - addMandatoryFacetValues(results, RuleIndex.FACET_REPOSITORIES, request.paramAsStrings(PARAM_REPOSITORIES)); - addMandatoryFacetValues(results, RuleIndex.FACET_STATUSES, RuleIndex.ALL_STATUSES_EXCEPT_REMOVED); - addMandatoryFacetValues(results, RuleIndex.FACET_SEVERITIES, Severity.ALL); - addMandatoryFacetValues(results, RuleIndex.FACET_ACTIVE_SEVERITIES, Severity.ALL); - addMandatoryFacetValues(results, RuleIndex.FACET_TAGS, request.paramAsStrings(PARAM_TAGS)); + addMandatoryFacetValues(results, FACET_LANGUAGES, request.paramAsStrings(PARAM_LANGUAGES)); + addMandatoryFacetValues(results, FACET_REPOSITORIES, request.paramAsStrings(PARAM_REPOSITORIES)); + addMandatoryFacetValues(results, FACET_STATUSES, ALL_STATUSES_EXCEPT_REMOVED); + addMandatoryFacetValues(results, FACET_SEVERITIES, Severity.ALL); + addMandatoryFacetValues(results, FACET_ACTIVE_SEVERITIES, Severity.ALL); + addMandatoryFacetValues(results, FACET_TAGS, request.paramAsStrings(PARAM_TAGS)); Common.Facet.Builder facet = Common.Facet.newBuilder(); Common.FacetValue.Builder value = Common.FacetValue.newBuilder(); diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java b/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java index 2f98fd19c53..25e4032fd42 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java @@ -30,7 +30,6 @@ import org.sonar.server.db.DeprecatedDao; import org.sonar.server.issue.index.IssueAuthorizationIndexer; import org.sonar.server.issue.index.IssueIndexer; import org.sonar.server.qualityprofile.index.ActiveRuleIndex; -import org.sonar.server.rule.index.RuleIndex; import org.sonar.server.test.index.TestIndexer; import org.sonar.server.user.index.UserIndexer; import org.sonar.server.view.index.ViewIndexer; @@ -56,7 +55,8 @@ public class IndexSynchronizer { */ public IndexSynchronizer(DbClient db, IndexClient index, TestIndexer testIndexer, IssueAuthorizationIndexer issueAuthorizationIndexer, IssueIndexer issueIndexer, - UserIndexer userIndexer, ViewIndexer viewIndexer, ActivityIndexer activityIndexer, Settings settings) { + UserIndexer userIndexer, ViewIndexer viewIndexer, ActivityIndexer activityIndexer, + Settings settings) { this.db = db; this.index = index; this.testIndexer = testIndexer; @@ -71,7 +71,7 @@ public class IndexSynchronizer { public void executeDeprecated() { DbSession session = db.openSession(false); try { - synchronize(session, db.deprecatedRuleDao(), index.get(RuleIndex.class)); + // synchronize(session, db.deprecatedRuleDao(), index.get(RuleIndex.class)); synchronize(session, db.activeRuleDao(), index.get(ActiveRuleIndex.class)); session.commit(); } finally { diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java b/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java index 02b9cdce9d2..36bbd9318a4 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java @@ -23,6 +23,11 @@ import com.google.common.base.Function; import com.google.common.base.Throwables; import com.google.common.collect.Collections2; import com.google.common.collect.Iterables; +import java.io.File; +import java.io.FileInputStream; +import java.util.Collections; +import java.util.List; +import java.util.Map; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.math.RandomUtils; import org.apache.commons.lang.reflect.ConstructorUtils; @@ -47,12 +52,6 @@ import org.sonar.server.search.BaseDoc; import org.sonar.server.search.SearchClient; import org.sonar.test.TestUtils; -import java.io.File; -import java.io.FileInputStream; -import java.util.Collections; -import java.util.List; -import java.util.Map; - import static com.google.common.collect.Lists.newArrayList; import static org.assertj.core.api.Assertions.assertThat; diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java index 49d27ae3656..e8ee95c0c1b 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java @@ -21,6 +21,7 @@ package org.sonar.server.es; import com.google.common.collect.ImmutableMap; import java.util.Map; +import org.assertj.core.data.MapEntry; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.junit.Test; @@ -66,8 +67,8 @@ public class NewIndexTest { mapping.createUuidPathField("uuid_path_field"); mapping = index.getTypes().get("issue"); - assertThat(mapping.getAttributes().get("dynamic")).isEqualTo("true"); assertThat(mapping).isNotNull(); + assertThat(mapping.getAttributes().get("dynamic")).isEqualTo("true"); assertThat(mapping.getProperty("foo_field")).isInstanceOf(Map.class); assertThat((Map) mapping.getProperty("foo_field")).containsEntry("type", "string"); assertThat((Map) mapping.getProperty("byte_field")).isNotEmpty(); @@ -184,4 +185,26 @@ public class NewIndexTest { assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_SHARDS)).isEqualTo("3"); assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("1"); } + + @Test + public void index_with_source() { + NewIndex index = new NewIndex("issues"); + NewIndex.NewIndexType mapping = index.createType("issue"); + mapping.setEnableSource(true); + + mapping = index.getTypes().get("issue"); + assertThat(mapping).isNotNull(); + assertThat((Map)mapping.getAttributes().get("_source")).containsExactly(MapEntry.entry("enabled", true)); + } + + @Test + public void index_without_source() { + NewIndex index = new NewIndex("issues"); + NewIndex.NewIndexType mapping = index.createType("issue"); + mapping.setEnableSource(false); + + mapping = index.getTypes().get("issue"); + assertThat(mapping).isNotNull(); + assertThat((Map)mapping.getAttributes().get("_source")).containsExactly(MapEntry.entry("enabled", false)); + } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/rule/RuleBackendMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/rule/RuleBackendMediumTest.java deleted file mode 100644 index b243bebe81d..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/rule/RuleBackendMediumTest.java +++ /dev/null @@ -1,418 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2016 SonarSource SA - * mailto:contact AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.rule; - -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import java.util.Collection; -import java.util.List; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.sonar.api.rule.RuleStatus; -import org.sonar.api.server.rule.RuleParamType; -import org.sonar.db.DbSession; -import org.sonar.db.rule.RuleDto; -import org.sonar.db.rule.RuleParamDto; -import org.sonar.db.rule.RuleTesting; -import org.sonar.server.db.DbClient; -import org.sonar.server.platform.Platform; -import org.sonar.server.rule.db.RuleDao; -import org.sonar.server.rule.index.RuleDoc; -import org.sonar.server.rule.index.RuleIndex; -import org.sonar.server.rule.index.RuleQuery; -import org.sonar.server.search.QueryContext; -import org.sonar.server.tester.ServerTester; -import org.sonar.server.tester.UserSessionRule; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * Test persistence in db and indexation in es (--> integration of DAOs and Indexes) - */ -public class RuleBackendMediumTest { - - @ClassRule - public static ServerTester tester = new ServerTester(); - @org.junit.Rule - public UserSessionRule userSessionRule = UserSessionRule.forServerTester(tester); - - RuleDao dao = tester.get(RuleDao.class); - RuleIndex index = tester.get(RuleIndex.class); - DbClient db; - DbSession dbSession; - - @Before - public void before() { - tester.clearDbAndIndexes(); - db = tester.get(DbClient.class); - dbSession = tester.get(DbClient.class).openSession(false); - } - - @After - public void after() { - dbSession.close(); - } - - @Test - public void insert_in_db_and_multiget_in_es() { - // insert db - RuleDto ruleDto = RuleTesting.newXooX1(); - RuleDto ruleDto2 = RuleTesting.newXooX2(); - dao.insert(dbSession, ruleDto, ruleDto2); - dbSession.commit(); - - // check that we get two rules - Collection hits = index.getByKeys(RuleTesting.XOO_X1, RuleTesting.XOO_X2); - assertThat(hits).hasSize(2); - } - - @Test - public void insert_in_db_and_index_in_es() { - // insert db - RuleDto ruleDto = RuleTesting.newXooX1(); - dao.insert(dbSession, ruleDto); - dbSession.commit(); - - // verify that rule is persisted in db - RuleDto persistedDto = dao.getNullableByKey(dbSession, RuleTesting.XOO_X1); - assertThat(persistedDto).isNotNull(); - assertThat(persistedDto.getId()).isGreaterThanOrEqualTo(0); - assertThat(persistedDto.getRuleKey()).isEqualTo(ruleDto.getRuleKey()); - assertThat(persistedDto.getLanguage()).isEqualTo(ruleDto.getLanguage()); - assertThat(persistedDto.getCreatedAt()).isNotNull(); - assertThat(persistedDto.getUpdatedAt()).isNotNull(); - - // verify that rule is indexed in es - Rule hit = index.getByKey(RuleTesting.XOO_X1); - assertRuleEquivalent(ruleDto, hit); - - // Verify Multi-get - Collection hits = index.getByKeys(RuleTesting.XOO_X1); - assertThat(hits).hasSize(1); - assertRuleEquivalent(ruleDto, Iterables.getFirst(hits, null)); - - } - - private void assertRuleEquivalent(RuleDto ruleDto, Rule hit) { - assertThat(hit).isNotNull(); - assertThat(hit.key().repository()).isEqualTo(ruleDto.getRepositoryKey()); - assertThat(hit.key().rule()).isEqualTo(ruleDto.getRuleKey()); - assertThat(hit.language()).isEqualTo(ruleDto.getLanguage()); - assertThat(hit.name()).isEqualTo(ruleDto.getName()); - assertThat(hit.htmlDescription()).isEqualTo(ruleDto.getDescription()); - assertThat(hit.status()).isEqualTo(RuleStatus.READY); - assertThat(hit.createdAt()).isNotNull(); - assertThat(hit.updatedAt()).isNotNull(); - assertThat(hit.internalKey()).isEqualTo(ruleDto.getConfigKey()); - assertThat(hit.severity()).isEqualTo(ruleDto.getSeverityString()); - assertThat(hit.isTemplate()).isFalse(); - assertThat(hit.effortToFixDescription()).isEqualTo(ruleDto.getEffortToFixDescription()); - } - - @Test - public void insert_rule_tags_in_db_and_index_in_es() { - // insert db - RuleDto ruleDto = RuleTesting.newXooX1(); - dao.insert(dbSession, ruleDto); - dbSession.commit(); - - RuleDto persistedDto = dao.getNullableByKey(dbSession, RuleTesting.XOO_X1); - assertThat(persistedDto.getTags().containsAll(ruleDto.getTags())).isTrue(); - assertThat(persistedDto.getSystemTags().containsAll(ruleDto.getSystemTags())).isTrue(); - - Rule hit = index.getByKey(RuleTesting.XOO_X1); - assertThat(hit.tags().containsAll(ruleDto.getTags())).isTrue(); - assertThat(hit.systemTags().containsAll(ruleDto.getSystemTags())).isTrue(); - } - - @Test - public void insert_and_index_rule_parameters() { - // insert db - RuleDto ruleDto = RuleTesting.newXooX1(); - dao.insert(dbSession, ruleDto); - dbSession.commit(); - - RuleParamDto minParamDto = new RuleParamDto() - .setName("min") - .setType(RuleParamType.INTEGER.type()) - .setDefaultValue("2") - .setDescription("Minimum"); - dao.insertRuleParam(dbSession, ruleDto, minParamDto); - RuleParamDto maxParamDto = new RuleParamDto() - .setName("max") - .setType(RuleParamType.INTEGER.type()) - .setDefaultValue("10") - .setDescription("Maximum"); - dao.insertRuleParam(dbSession, ruleDto, maxParamDto); - dbSession.commit(); - - // Verify that RuleDto has date from insertion - RuleDto theRule = dao.getNullableByKey(dbSession, RuleTesting.XOO_X1); - assertThat(theRule.getCreatedAt()).isNotNull(); - assertThat(theRule.getUpdatedAt()).isNotNull(); - - // verify that parameters are persisted in db - List persistedDtos = dao.selectRuleParamsByRuleKey(dbSession, theRule.getKey()); - assertThat(persistedDtos).hasSize(2); - - // verify that parameters are indexed in es - - Rule hit = index.getByKey(RuleTesting.XOO_X1); - assertThat(hit).isNotNull(); - assertThat(hit.key()).isNotNull(); - - RuleService service = tester.get(RuleService.class); - Rule rule = service.getByKey(RuleTesting.XOO_X1); - - assertThat(rule.params()).hasSize(2); - assertThat(Iterables.getLast(rule.params(), null).key()).isEqualTo("max"); - } - - @Test - public void insert_and_delete_rule_parameters() { - // insert db - RuleDto ruleDto = RuleTesting.newXooX1(); - dao.insert(dbSession, ruleDto); - dbSession.commit(); - - RuleParamDto minParamDto = new RuleParamDto() - .setName("min") - .setType(RuleParamType.INTEGER.type()) - .setDefaultValue("2") - .setDescription("Minimum"); - dao.insertRuleParam(dbSession, ruleDto, minParamDto); - RuleParamDto maxParamDto = new RuleParamDto() - .setName("max") - .setType(RuleParamType.INTEGER.type()) - .setDefaultValue("10") - .setDescription("Maximum"); - dao.insertRuleParam(dbSession, ruleDto, maxParamDto); - dbSession.commit(); - - // 0. Verify that RuleDto has date from insertion - assertThat(dao.selectRuleParamsByRuleKey(dbSession, RuleTesting.XOO_X1)).hasSize(2); - assertThat(index.getByKey(RuleTesting.XOO_X1).params()).hasSize(2); - - // 1. Delete parameter - dao.deleteRuleParam(dbSession, ruleDto, maxParamDto); - dbSession.commit(); - - // 2. assert only one param left - assertThat(dao.selectRuleParamsByRuleKey(dbSession, RuleTesting.XOO_X1)).hasSize(1); - assertThat(index.getByKey(RuleTesting.XOO_X1).params()).hasSize(1); - } - - @Test - public void insert_and_update_rule() { - // insert db - RuleDto ruleDto = RuleTesting.newXooX1() - .setTags(ImmutableSet.of("hello")) - .setName("first name"); - dao.insert(dbSession, ruleDto); - dbSession.commit(); - - // verify that parameters are indexed in es - - Rule hit = index.getByKey(RuleTesting.XOO_X1); - assertThat(hit.tags()).containsExactly("hello"); - assertThat(hit.name()).isEqualTo("first name"); - - // Update in DB - ruleDto.setTags(ImmutableSet.of("world")) - .setName("second name"); - dao.update(dbSession, ruleDto); - dbSession.commit(); - - // verify that parameters are updated in es - - hit = index.getByKey(RuleTesting.XOO_X1); - assertThat(hit.tags()).containsExactly("world"); - assertThat(hit.name()).isEqualTo("second name"); - } - - @Test - public void insert_and_update_rule_param() { - - // insert db - RuleDto ruleDto = RuleTesting.newXooX1(); - dao.insert(dbSession, ruleDto); - dbSession.commit(); - - RuleParamDto minParamDto = new RuleParamDto() - .setName("min") - .setType(RuleParamType.INTEGER.type()) - .setDefaultValue("2") - .setDescription("Minimum"); - dao.insertRuleParam(dbSession, ruleDto, minParamDto); - - RuleParamDto maxParamDto = new RuleParamDto() - .setName("max") - .setType(RuleParamType.INTEGER.type()) - .setDefaultValue("10") - .setDescription("Maximum"); - dao.insertRuleParam(dbSession, ruleDto, maxParamDto); - dbSession.commit(); - - // verify that parameters are indexed in es - - Rule hit = index.getByKey(RuleTesting.XOO_X1); - assertThat(hit.params()).hasSize(2); - - RuleParam param = hit.params().get(0); - assertThat(param.key()).isEqualTo("min"); - assertThat(param.defaultValue()).isEqualTo("2"); - assertThat(param.description()).isEqualTo("Minimum"); - - // Update in DB - minParamDto - .setDefaultValue("0.5") - .setDescription("new description"); - dao.updateRuleParam(dbSession, ruleDto, minParamDto); - dbSession.commit(); - - // verify that parameters are updated in es - - hit = index.getByKey(RuleTesting.XOO_X1); - assertThat(hit.params()).hasSize(2); - - param = null; - for (RuleParam pparam : hit.params()) { - if (pparam.key().equals("min")) { - param = pparam; - } - } - assertThat(param).isNotNull(); - assertThat(param.key()).isEqualTo("min"); - assertThat(param.defaultValue()).isEqualTo("0.5"); - assertThat(param.description()).isEqualTo("new description"); - } - - @Test - @Deprecated - public void has_id() { - - RuleDto ruleDto = RuleTesting.newXooX1(); - dao.insert(dbSession, ruleDto); - dbSession.commit(); - - assertThat(((RuleDoc) index.getByKey(RuleTesting.XOO_X1)).id()).isEqualTo(ruleDto.getId()); - } - - @Test - public void insert_update_debt_overload() { - - RuleDto ruleDto = RuleTesting.newXooX1() - .setRemediationFunction(null) - .setRemediationCoefficient(null) - .setRemediationOffset(null); - - RuleDto overloadedRuleDto = RuleTesting.newXooX2(); - - dao.insert(dbSession, ruleDto, overloadedRuleDto); - dbSession.commit(); - - // Assert is overloaded or not - assertThat(index.getByKey(RuleTesting.XOO_X1).debtOverloaded()).isFalse(); - assertThat(index.getByKey(RuleTesting.XOO_X2).debtOverloaded()).isTrue(); - - // Assert overloaded value - Rule base = index.getByKey(RuleTesting.XOO_X1); - Rule overloaded = index.getByKey(RuleTesting.XOO_X2); - - assertThat(base.debtRemediationFunction().type().toString()) - .isEqualTo(ruleDto.getDefaultRemediationFunction()); - assertThat(base.debtRemediationFunction().coefficient()) - .isEqualTo(ruleDto.getDefaultRemediationCoefficient()); - assertThat(base.debtRemediationFunction().offset()) - .isEqualTo(ruleDto.getDefaultRemediationOffset()); - - assertThat(overloaded.debtRemediationFunction().type().toString()) - .isEqualTo(overloadedRuleDto.getRemediationFunction()); - assertThat(overloaded.debtRemediationFunction().coefficient()) - .isEqualTo(overloadedRuleDto.getRemediationCoefficient()); - assertThat(overloaded.debtRemediationFunction().offset()) - .isEqualTo(overloadedRuleDto.getRemediationOffset()); - } - - @Test - public void should_not_find_removed() { - // insert db - dao.insert(dbSession, - RuleTesting.newXooX1(), - RuleTesting.newXooX2().setStatus(RuleStatus.REMOVED)); - dbSession.commit(); - - // 0. Assert rules are in DB - assertThat(dao.selectAll(dbSession)).hasSize(2); - - // 1. assert getBy for removed - assertThat(index.getByKey(RuleTesting.XOO_X2)).isNotNull(); - - // 2. assert find does not get REMOVED - List rules = index.search(new RuleQuery(), new QueryContext(userSessionRule)).getHits(); - assertThat(rules).hasSize(1); - assertThat(rules.get(0).key()).isEqualTo(RuleTesting.XOO_X1); - } - - @Test - public void synchronize_after() { - // insert db - dao.insert(dbSession, - RuleTesting.newXooX1()); - dbSession.commit(); - - // 0. Assert rules are in DB - assertThat(dao.selectAll(dbSession)).hasSize(1); - assertThat(index.countAll()).isEqualTo(1); - - tester.clearIndexes(); - assertThat(index.countAll()).isEqualTo(0); - - tester.get(Platform.class).executeStartupTasks(); - assertThat(index.countAll()).isEqualTo(1); - - } - - @Test - public void synchronize_after_with_nested() { - RuleDto rule = RuleTesting.newXooX1(); - - // insert db - dao.insert(dbSession, rule); - - dao.insertRuleParam(dbSession, rule, RuleParamDto.createFor(rule).setName("MyParam").setType("STRING").setDefaultValue("test")); - dbSession.commit(); - - // 0. Assert rules are in DB - assertThat(dao.selectAll(dbSession)).hasSize(1); - assertThat(index.countAll()).isEqualTo(1); - - tester.clearIndexes(); - assertThat(index.countAll()).isEqualTo(0); - - tester.get(Platform.class).executeStartupTasks(); - assertThat(index.countAll()).isEqualTo(1); - - assertThat(index.getByKey(rule.getKey()).param("MyParam").defaultValue()).isEqualTo("test"); - - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleDocTesting.java b/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleDocTesting.java new file mode 100644 index 00000000000..9b02c1c573c --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleDocTesting.java @@ -0,0 +1,50 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.rule.index; + +import com.google.common.collect.Maps; +import java.util.Arrays; +import org.sonar.api.rule.RuleKey; +import org.sonar.api.rule.RuleStatus; +import org.sonar.api.rule.Severity; +import org.sonar.db.rule.RuleTesting; + +public class RuleDocTesting { + + public static RuleDoc newDoc() { + return newDoc(RuleTesting.XOO_X1); + } + + public static RuleDoc newDoc(RuleKey ruleKey) { + return new RuleDoc(Maps.newHashMap()) + .setKey(ruleKey.toString()) + .setRepository(ruleKey.repository()) + .setRuleKey(ruleKey.rule()) + .setName("Name " + ruleKey.toString()) + .setHtmlDescription("Description " + ruleKey.rule()) + .setSeverity(Severity.CRITICAL) + .setStatus(RuleStatus.READY.name()) + .setLanguage("xoo") + .setIsTemplate(false) + .setAllTags(Arrays.asList("bug", "performance")) + .setCreatedAt(150000000L) + .setUpdatedAt(160000000L); + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndex2Test.java b/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndex2Test.java new file mode 100644 index 00000000000..27c1deef067 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndex2Test.java @@ -0,0 +1,732 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.rule.index; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.api.config.Settings; +import org.sonar.api.rule.RuleKey; +import org.sonar.api.rule.RuleStatus; +import org.sonar.server.es.EsTester; +import org.sonar.server.es.SearchIdResult; +import org.sonar.server.es.SearchOptions; +import org.sonar.server.tester.UserSessionRule; + +import static java.util.Arrays.asList; +import static java.util.Collections.singleton; +import static java.util.Collections.singletonList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.data.MapEntry.entry; +import static org.junit.Assert.fail; +import static org.sonar.api.rule.Severity.BLOCKER; +import static org.sonar.api.rule.Severity.INFO; +import static org.sonar.api.rule.Severity.MINOR; +import static org.sonar.server.rule.index.RuleDocTesting.newDoc; +import static org.sonar.server.rule.index.RuleIndex2.FACET_LANGUAGES; +import static org.sonar.server.rule.index.RuleIndex2.FACET_REPOSITORIES; +import static org.sonar.server.rule.index.RuleIndex2.FACET_TAGS; + +public class RuleIndex2Test { + + @ClassRule + public static EsTester tester = new EsTester().addDefinitions(new RuleIndexDefinition(new Settings())); + + @Rule + public UserSessionRule userSessionRule = UserSessionRule.standalone(); + + RuleIndex2 index; + + RuleIndexer ruleIndexer; + + @Before + public void setUp() { + tester.truncateIndices(); + ruleIndexer = new RuleIndexer(null, tester.client()); + index = new RuleIndex2(tester.client()); + } + + @Test + public void search_all_rules() { + indexRules( + newDoc(RuleKey.of("javascript", "S001")), + newDoc(RuleKey.of("java", "S002"))); + + SearchIdResult results = index.search(new RuleQuery(), new SearchOptions()); + + assertThat(results.getTotal()).isEqualTo(2); + assertThat(results.getIds()).hasSize(2); + } + + @Test + public void search_key_by_query() { + indexRules( + newDoc(RuleKey.of("javascript", "X001")), + newDoc(RuleKey.of("cobol", "X001")), + newDoc(RuleKey.of("php", "S002"))); + + // key + RuleQuery query = new RuleQuery().setQueryText("X001"); + assertThat(index.search(query, new SearchOptions()).getIds()).hasSize(2); + + // partial key does not match + query = new RuleQuery().setQueryText("X00"); + assertThat(index.search(query, new SearchOptions()).getIds()).isEmpty(); + + // repo:key -> nice-to-have ! + query = new RuleQuery().setQueryText("javascript:X001"); + assertThat(index.search(query, new SearchOptions()).getIds()).hasSize(1); + } + + @Test + public void filter_by_key() { + indexRules( + newDoc(RuleKey.of("javascript", "X001")), + newDoc(RuleKey.of("cobol", "X001")), + newDoc(RuleKey.of("php", "S002"))); + + // key + RuleQuery query = new RuleQuery().setKey(RuleKey.of("javascript", "X001").toString()); + + assertThat(index.search(query, new SearchOptions()).getIds()).hasSize(1); + + // partial key does not match + query = new RuleQuery().setKey("X001"); + assertThat(index.search(query, new SearchOptions()).getIds()).isEmpty(); + } + + @Test + public void search_name_by_query() { + indexRules(newDoc(RuleKey.of("javascript", "S001")) + .setName("testing the partial match and matching of rule")); + + // substring + RuleQuery query = new RuleQuery().setQueryText("test"); + assertThat(index.search(query, new SearchOptions()).getIds()).hasSize(1); + + // substring + query = new RuleQuery().setQueryText("partial match"); + assertThat(index.search(query, new SearchOptions()).getIds()).hasSize(1); + + // case-insensitive + query = new RuleQuery().setQueryText("TESTING"); + assertThat(index.search(query, new SearchOptions()).getIds()).hasSize(1); + + // not found + query = new RuleQuery().setQueryText("not present"); + assertThat(index.search(query, new SearchOptions()).getIds()).isEmpty(); + } + + @Test + public void search_name_with_protected_chars() { + String nameWithProtectedChars = "ja#va&sc\"r:ipt"; + + indexRules(newDoc(RuleKey.of("javascript", "S001")) + .setName(nameWithProtectedChars)); + + RuleQuery protectedCharsQuery = new RuleQuery().setQueryText(nameWithProtectedChars); + List results = index.search(protectedCharsQuery, new SearchOptions()).getIds(); + assertThat(results).containsOnly(RuleKey.of("javascript", "S001")); + } + + @Test + public void search_by_any_of_repositories() { + indexRules( + newDoc(RuleKey.of("findbugs", "S001")), + newDoc(RuleKey.of("pmd", "S002"))); + + RuleQuery query = new RuleQuery().setRepositories(asList("checkstyle", "pmd")); + SearchIdResult results = index.search(query, new SearchOptions()); + assertThat(results.getIds()).containsOnly(RuleKey.of("pmd", "S002")); + + // no results + query = new RuleQuery().setRepositories(singletonList("checkstyle")); + assertThat(index.search(query, new SearchOptions()).getIds()).isEmpty(); + + // empty list => no filter + query = new RuleQuery().setRepositories(Collections.emptyList()); + assertThat(index.search(query, new SearchOptions()).getIds()).hasSize(2); + } + + @Test + public void search_by_tag() { + indexRules( + newDoc(RuleKey.of("java", "S001")).setAllTags(singleton("tag1")), + newDoc(RuleKey.of("java", "S002")).setAllTags(singleton("tag2"))); + + // find all + RuleQuery query = new RuleQuery(); + assertThat(index.search(query, new SearchOptions()).getIds()).hasSize(2); + + // tag1 in query + query = new RuleQuery().setQueryText("tag1"); + assertThat(index.search(query, new SearchOptions()).getIds()).containsOnly(RuleKey.of("java", "S001")); + + // tag1 and tag2 in query + query = new RuleQuery().setQueryText("tag1 tag2"); + assertThat(index.search(query, new SearchOptions()).getIds()).hasSize(2); + + // tag2 in filter + query = new RuleQuery().setTags(ImmutableSet.of("tag2")); + assertThat(index.search(query, new SearchOptions()).getIds()).containsOnly(RuleKey.of("java", "S002")); + + // tag2 in filter and tag1 tag2 in query + query = new RuleQuery().setTags(ImmutableSet.of("tag2")).setQueryText("tag1"); + assertThat(index.search(query, new SearchOptions()).getIds()).hasSize(0); + + // tag2 in filter and tag1 in query + query = new RuleQuery().setTags(ImmutableSet.of("tag2")).setQueryText("tag1 tag2"); + assertThat(index.search(query, new SearchOptions()).getIds()).containsOnly(RuleKey.of("java", "S002")); + + // null list => no filter + query = new RuleQuery().setTags(Collections.emptySet()); + assertThat(index.search(query, new SearchOptions()).getIds()).hasSize(2); + + // null list => no filter + query = new RuleQuery().setTags(null); + assertThat(index.search(query, new SearchOptions()).getIds()).hasSize(2); + } + + @Test + public void search_by_is_template() { + indexRules( + newDoc(RuleKey.of("java", "S001")).setIsTemplate(false), + newDoc(RuleKey.of("java", "S002")).setIsTemplate(true)); + + // find all + RuleQuery query = new RuleQuery(); + SearchIdResult results = index.search(query, new SearchOptions()); + assertThat(results.getIds()).hasSize(2); + + // Only template + query = new RuleQuery().setIsTemplate(true); + results = index.search(query, new SearchOptions()); + assertThat(results.getIds()).containsOnly(RuleKey.of("java", "S002")); + + // Only not template + query = new RuleQuery().setIsTemplate(false); + results = index.search(query, new SearchOptions()); + assertThat(results.getIds()).containsOnly(RuleKey.of("java", "S001")); + + // null => no filter + query = new RuleQuery().setIsTemplate(null); + assertThat(index.search(query, new SearchOptions()).getIds()).hasSize(2); + } + + @Test + public void search_by_template_key() { + indexRules( + newDoc(RuleKey.of("java", "S001")).setIsTemplate(true), + newDoc(RuleKey.of("java", "S001_MY_CUSTOM")).setTemplateKey("java:S001")); + + // find all + RuleQuery query = new RuleQuery(); + SearchIdResult results = index.search(query, new SearchOptions()); + assertThat(results.getIds()).hasSize(2); + + // Only custom rule + query = new RuleQuery().setTemplateKey("java:S001"); + results = index.search(query, new SearchOptions()); + assertThat(results.getIds()).containsOnly(RuleKey.of("java", "S001_MY_CUSTOM")); + + // null => no filter + query = new RuleQuery().setTemplateKey(null); + assertThat(index.search(query, new SearchOptions()).getIds()).hasSize(2); + } + + @Test + public void search_by_any_of_languages() { + indexRules( + newDoc(RuleKey.of("java", "S001")).setLanguage("java"), + newDoc(RuleKey.of("javascript", "S002")).setLanguage("js")); + + RuleQuery query = new RuleQuery().setLanguages(asList("cobol", "js")); + SearchIdResult results = index.search(query, new SearchOptions()); + assertThat(results.getIds()).containsOnly(RuleKey.of("javascript", "S002")); + + // no results + query = new RuleQuery().setLanguages(singletonList("cpp")); + assertThat(index.search(query, new SearchOptions()).getIds()).isEmpty(); + + // empty list => no filter + query = new RuleQuery().setLanguages(Collections.emptyList()); + assertThat(index.search(query, new SearchOptions()).getIds()).hasSize(2); + + // null list => no filter + query = new RuleQuery().setLanguages(null); + assertThat(index.search(query, new SearchOptions()).getIds()).hasSize(2); + } + + @Test + public void search_by_any_of_severities() { + indexRules( + newDoc(RuleKey.of("java", "S001")).setSeverity(BLOCKER), + newDoc(RuleKey.of("java", "S002")).setSeverity(INFO)); + + RuleQuery query = new RuleQuery().setSeverities(asList(INFO, MINOR)); + SearchIdResult results = index.search(query, new SearchOptions()); + assertThat(results.getIds()).containsOnly(RuleKey.of("java", "S002")); + + // no results + query = new RuleQuery().setSeverities(singletonList(MINOR)); + assertThat(index.search(query, new SearchOptions()).getIds()).isEmpty(); + + // empty list => no filter + query = new RuleQuery().setSeverities(Collections.emptyList()); + assertThat(index.search(query, new SearchOptions()).getIds()).hasSize(2); + + // null list => no filter + query = new RuleQuery().setSeverities(null); + assertThat(index.search(query, new SearchOptions()).getIds()).hasSize(2); + } + + @Test + public void search_by_any_of_statuses() { + indexRules( + newDoc(RuleKey.of("java", "S001")).setStatus(RuleStatus.BETA.name()), + newDoc(RuleKey.of("java", "S002")).setStatus(RuleStatus.READY.name())); + + RuleQuery query = new RuleQuery().setStatuses(asList(RuleStatus.DEPRECATED, RuleStatus.READY)); + SearchIdResult results = index.search(query, new SearchOptions()); + assertThat(results.getIds()).containsOnly(RuleKey.of("java", "S002")); + + // no results + query = new RuleQuery().setStatuses(singletonList(RuleStatus.DEPRECATED)); + assertThat(index.search(query, new SearchOptions()).getIds()).isEmpty(); + + // empty list => no filter + query = new RuleQuery().setStatuses(Collections.emptyList()); + assertThat(index.search(query, new SearchOptions()).getIds()).hasSize(2); + + // null list => no filter + query = new RuleQuery().setStatuses(null); + assertThat(index.search(query, new SearchOptions()).getIds()).hasSize(2); + } + + @Test + @Ignore + public void search_by_profile() { + // QualityProfileDto qualityProfileDto1 = QProfileTesting.newXooP1(); + // QualityProfileDto qualityProfileDto2 = QProfileTesting.newXooP2(); + // db.qualityProfileDao().insert(dbSession, qualityProfileDto1, qualityProfileDto2); + // + // RuleDto rule1 = RuleTesting.newXooX1(); + // RuleDto rule2 = RuleTesting.newXooX2(); + // RuleDto rule3 = RuleTesting.newXooX3(); + // dao.insert(dbSession, rule1, rule2, rule3); + // + // db.activeRuleDao().insert( + // dbSession, + // ActiveRuleDto.createFor(qualityProfileDto1, rule1).setSeverity("BLOCKER"), + // ActiveRuleDto.createFor(qualityProfileDto2, rule1).setSeverity("BLOCKER"), + // ActiveRuleDto.createFor(qualityProfileDto1, rule2).setSeverity("BLOCKER")); + // dbSession.commit(); + // dbSession.clearCache(); + // + // // 1. get all active rules. + // Result result = index.search(new RuleQuery().setActivation(true), + // new SearchOptions()); + // assertThat(result.getHits()).hasSize(2); + // + // // 2. get all inactive rules. + // result = index.search(new RuleQuery().setActivation(false), + // new SearchOptions()); + // assertThat(result.getHits()).hasSize(1); + // assertThat(result.getHits().get(0).name()).isEqualTo(rule3.getName()); + // + // // 3. get all rules not active on profile + // index.search(new RuleQuery().setActivation(false).setQProfileKey(qualityProfileDto2.getKey()), + // new SearchOptions()); + // // TODO + // assertThat(result.getHits()).hasSize(1); + // + // // 4. get all active rules on profile + // result = index.search(new RuleQuery().setActivation(true) + // .setQProfileKey(qualityProfileDto2.getKey()), + // new SearchOptions()); + // assertThat(result.getHits()).hasSize(1); + // assertThat(result.getHits().get(0).name()).isEqualTo(rule1.getName()); + } + + @Test + @Ignore + public void search_by_profile_and_inheritance() { + // QualityProfileDto qualityProfileDto1 = QProfileTesting.newXooP1(); + // QualityProfileDto qualityProfileDto2 = QProfileTesting.newXooP2().setParentKee(QProfileTesting.XOO_P1_KEY); + // db.qualityProfileDao().insert(dbSession, qualityProfileDto1, qualityProfileDto2); + // + // RuleDto rule1 = RuleTesting.newDto(RuleKey.of("xoo", "S001")); + // RuleDto rule2 = RuleTesting.newDto(RuleKey.of("xoo", "S002")); + // RuleDto rule3 = RuleTesting.newDto(RuleKey.of("xoo", "S003")); + // RuleDto rule4 = RuleTesting.newDto(RuleKey.of("xoo", "S004")); + // dao.insert(dbSession, rule1, rule2, rule3, rule4); + // + // db.activeRuleDao().insert( + // dbSession, + // ActiveRuleDto.createFor(qualityProfileDto1, rule1) + // .setSeverity("BLOCKER"), + // ActiveRuleDto.createFor(qualityProfileDto1, rule2) + // .setSeverity("BLOCKER"), + // ActiveRuleDto.createFor(qualityProfileDto1, rule3) + // .setSeverity("BLOCKER"), + // + // ActiveRuleDto.createFor(qualityProfileDto2, rule1) + // .setSeverity("MINOR") + // .setInheritance(ActiveRule.Inheritance.INHERITED.name()), + // ActiveRuleDto.createFor(qualityProfileDto2, rule2) + // .setSeverity("BLOCKER") + // .setInheritance(ActiveRule.Inheritance.OVERRIDES.name()), + // ActiveRuleDto.createFor(qualityProfileDto2, rule3) + // .setSeverity("BLOCKER") + // .setInheritance(ActiveRule.Inheritance.INHERITED.name()) + // ); + // + // dbSession.commit(); + // + // // 0. get all rules + // Result result = index.search(new RuleQuery(), + // new SearchOptions()); + // assertThat(result.getHits()).hasSize(4); + // + // // 1. get all active rules + // result = index.search(new RuleQuery().setActivation(true), + // new SearchOptions()); + // assertThat(result.getHits()).hasSize(3); + // + // // 2. get all inactive rules. + // result = index.search(new RuleQuery().setActivation(false), + // new SearchOptions()); + // assertThat(result.getHits()).hasSize(1); + // assertThat(result.getHits().get(0).name()).isEqualTo(rule4.getName()); + // + // // 3. get Inherited Rules on profile1 + // result = index.search(new RuleQuery().setActivation(true) + // .setQProfileKey(qualityProfileDto1.getKey()) + // .setInheritance(ImmutableSet.of(ActiveRule.Inheritance.INHERITED.name())), + // new SearchOptions() + // ); + // assertThat(result.getHits()).hasSize(0); + // + // // 4. get Inherited Rules on profile2 + // result = index.search(new RuleQuery().setActivation(true) + // .setQProfileKey(qualityProfileDto2.getKey()) + // .setInheritance(ImmutableSet.of(ActiveRule.Inheritance.INHERITED.name())), + // new SearchOptions() + // ); + // assertThat(result.getHits()).hasSize(2); + // + // // 5. get Overridden Rules on profile1 + // result = index.search(new RuleQuery().setActivation(true) + // .setQProfileKey(qualityProfileDto1.getKey()) + // .setInheritance(ImmutableSet.of(ActiveRule.Inheritance.OVERRIDES.name())), + // new SearchOptions() + // ); + // assertThat(result.getHits()).hasSize(0); + // + // // 6. get Overridden Rules on profile2 + // result = index.search(new RuleQuery().setActivation(true) + // .setQProfileKey(qualityProfileDto2.getKey()) + // .setInheritance(ImmutableSet.of(ActiveRule.Inheritance.OVERRIDES.name())), + // new SearchOptions() + // ); + // assertThat(result.getHits()).hasSize(1); + // + // // 7. get Inherited AND Overridden Rules on profile1 + // result = index.search(new RuleQuery().setActivation(true) + // .setQProfileKey(qualityProfileDto1.getKey()) + // .setInheritance(ImmutableSet.of( + // ActiveRule.Inheritance.INHERITED.name(), ActiveRule.Inheritance.OVERRIDES.name())), + // new SearchOptions() + // ); + // assertThat(result.getHits()).hasSize(0); + // + // // 8. get Inherited AND Overridden Rules on profile2 + // result = index.search(new RuleQuery().setActivation(true) + // .setQProfileKey(qualityProfileDto2.getKey()) + // .setInheritance(ImmutableSet.of( + // ActiveRule.Inheritance.INHERITED.name(), ActiveRule.Inheritance.OVERRIDES.name())), + // new SearchOptions() + // ); + // assertThat(result.getHits()).hasSize(3); + } + + @Test + @Ignore + public void search_by_profile_and_active_severity() { + // QualityProfileDto qualityProfileDto1 = QProfileTesting.newXooP1(); + // QualityProfileDto qualityProfileDto2 = QProfileTesting.newXooP2(); + // db.qualityProfileDao().insert(dbSession, qualityProfileDto1, qualityProfileDto2); + // + // RuleDto rule1 = RuleTesting.newXooX1().setSeverity("MAJOR"); + // RuleDto rule2 = RuleTesting.newXooX2().setSeverity("MINOR"); + // RuleDto rule3 = RuleTesting.newXooX3().setSeverity("INFO"); + // dao.insert(dbSession, rule1, rule2, rule3); + // + // db.activeRuleDao().insert( + // dbSession, + // ActiveRuleDto.createFor(qualityProfileDto1, rule1).setSeverity("BLOCKER"), + // ActiveRuleDto.createFor(qualityProfileDto2, rule1).setSeverity("BLOCKER"), + // ActiveRuleDto.createFor(qualityProfileDto1, rule2).setSeverity("CRITICAL")); + // dbSession.commit(); + // dbSession.clearCache(); + // + // // 1. get all active rules. + // Result result = index.search(new + // RuleQuery().setActivation(true).setQProfileKey(qualityProfileDto1.getKey()), + // new SearchOptions()); + // assertThat(result.getHits()).hasSize(2); + // + // // 2. get rules with active severity critical. + // result = index.search(new + // RuleQuery().setActivation(true).setQProfileKey(qualityProfileDto1.getKey()).setActiveSeverities(Arrays.asList("CRITICAL")), + // new SearchOptions().addFacets(Arrays.asList(RuleIndex.FACET_ACTIVE_SEVERITIES))); + // assertThat(result.getHits()).hasSize(1); + // assertThat(result.getHits().get(0).name()).isEqualTo(rule2.getName()); + // // check stickyness of active severity facet + // assertThat(result.getFacetValues(RuleIndex.FACET_ACTIVE_SEVERITIES)).containsOnly(new FacetValue("BLOCKER", 1), new + // FacetValue("CRITICAL", 1)); + // + // // 3. count activation severities of all active rules + // result = index.search(new RuleQuery(), + // new SearchOptions().addFacets(Arrays.asList(RuleIndex.FACET_ACTIVE_SEVERITIES))); + // assertThat(result.getHits()).hasSize(3); + // assertThat(result.getFacetValues(RuleIndex.FACET_ACTIVE_SEVERITIES)).containsOnly(new FacetValue("BLOCKER", 2), new + // FacetValue("CRITICAL", 1)); + } + + @Test + public void all_tags() { + indexRules( + newDoc(RuleKey.of("java", "S001")).setAllTags(asList("tag1", "sys1", "sys2")), + newDoc(RuleKey.of("java", "S002")).setAllTags(asList("tag2"))); + + assertThat(index.terms(RuleIndexDefinition.FIELD_RULE_ALL_TAGS, null, 10)).containsOnly("tag1", "tag2", "sys1", "sys2"); + } + + @Test + public void available_since() throws InterruptedException { + indexRules( + newDoc(RuleKey.of("java", "S001")).setCreatedAt(1000L), + newDoc(RuleKey.of("java", "S002")).setCreatedAt(2000L)); + + // 0. find all rules; + assertThat(index.search(new RuleQuery(), new SearchOptions()).getIds()).hasSize(2); + + // 1. find all rules available since a date; + RuleQuery availableSinceQuery = new RuleQuery().setAvailableSince(2000L); + assertThat(index.search(availableSinceQuery, new SearchOptions()).getIds()).containsOnly(RuleKey.of("java", "S002")); + + // 2. find no new rules since tomorrow. + RuleQuery availableSinceNowQuery = new RuleQuery().setAvailableSince(3000L); + assertThat(index.search(availableSinceNowQuery, new SearchOptions()).getIds()).hasSize(0); + } + + @Test + public void global_facet_on_repositories_and_tags() { + indexRules( + newDoc(RuleKey.of("php", "S001")).setAllTags(singletonList("sysTag")), + newDoc(RuleKey.of("php", "S002")).setAllTags(singletonList("tag1")), + newDoc(RuleKey.of("javascript", "S002")).setAllTags(asList("tag1", "tag2"))); + + // should not have any facet! + RuleQuery query = new RuleQuery(); + SearchIdResult result = index.search(query, new SearchOptions()); + assertThat(result.getFacets().getAll()).isEmpty(); + + // should not have any facet on non matching query! + result = index.search(new RuleQuery().setQueryText("aeiou"), new SearchOptions().addFacets(singletonList("repositories"))); + assertThat(result.getFacets().getAll()).hasSize(1); + assertThat(result.getFacets().getAll().get("repositories")).isEmpty(); + + // Repositories Facet is preset + result = index.search(query, new SearchOptions().addFacets(asList("repositories", "tags"))); + assertThat(result.getFacets()).isNotNull(); + assertThat(result.getFacets().getAll()).hasSize(2); + + // Verify the value of a given facet + Map repoFacets = result.getFacets().get("repositories"); + assertThat(repoFacets).containsOnly(entry("php", 2L), entry("javascript", 1L)); + + // Check that tag facet has both Tags and SystemTags values + Map tagFacets = result.getFacets().get("tags"); + assertThat(tagFacets).containsOnly(entry("tag1", 2L), entry("sysTag", 1L), entry("tag2", 1L)); + } + + @Test + public void sticky_facets() { + indexRules( + newDoc(RuleKey.of("xoo", "S001")).setLanguage("java").setAllTags(Collections.emptyList()), + newDoc(RuleKey.of("xoo", "S002")).setLanguage("java").setAllTags(Collections.emptyList()), + newDoc(RuleKey.of("xoo", "S003")).setLanguage("java").setAllTags(asList("T1", "T2")), + newDoc(RuleKey.of("xoo", "S011")).setLanguage("cobol").setAllTags(Collections.emptyList()), + newDoc(RuleKey.of("xoo", "S012")).setLanguage("cobol").setAllTags(Collections.emptyList()), + newDoc(RuleKey.of("foo", "S013")).setLanguage("cobol").setAllTags(asList("T3", "T4")), + newDoc(RuleKey.of("foo", "S111")).setLanguage("cpp").setAllTags(Collections.emptyList()), + newDoc(RuleKey.of("foo", "S112")).setLanguage("cpp").setAllTags(Collections.emptyList()), + newDoc(RuleKey.of("foo", "S113")).setLanguage("cpp").setAllTags(asList("T2", "T3"))); + + // 0 assert Base + assertThat(index.search(new RuleQuery(), new SearchOptions()).getIds()).hasSize(9); + + // 1 Facet with no filters at all + SearchIdResult result = index.search(new RuleQuery(), new SearchOptions().addFacets(asList("languages", "repositories", "tags"))); + assertThat(result.getFacets().getAll()).hasSize(3); + assertThat(result.getFacets().getAll().get(FACET_LANGUAGES).keySet()).containsOnly("cpp", "java", "cobol"); + assertThat(result.getFacets().getAll().get(FACET_REPOSITORIES).keySet()).containsOnly("xoo", "foo"); + assertThat(result.getFacets().getAll().get(FACET_TAGS).keySet()).containsOnly("T1", "T2", "T3", "T4"); + + // 2 Facet with a language filter + // -- lang facet should still have all language + result = index.search(new RuleQuery().setLanguages(ImmutableList.of("cpp")) + , new SearchOptions().addFacets(asList("languages", "repositories", "tags"))); + assertThat(result.getIds()).hasSize(3); + assertThat(result.getFacets().getAll()).hasSize(3); + assertThat(result.getFacets().get(FACET_LANGUAGES).keySet()).containsOnly("cpp", "java", "cobol"); + + // 3 facet with 2 filters + // -- lang facet for tag T2 + // -- tag facet for lang cpp + // -- repository for cpp & T2 + result = index.search(new RuleQuery() + .setLanguages(ImmutableList.of("cpp")) + .setTags(ImmutableList.of("T2")) + , new SearchOptions().addFacets(asList("languages", "repositories", "tags"))); + assertThat(result.getIds()).hasSize(1); + assertThat(result.getFacets().getAll()).hasSize(3); + assertThat(result.getFacets().get(FACET_LANGUAGES).keySet()).containsOnly("cpp", "java"); + assertThat(result.getFacets().get(FACET_REPOSITORIES).keySet()).containsOnly("foo"); + assertThat(result.getFacets().get(FACET_TAGS).keySet()).containsOnly("T2", "T3"); + + // 4 facet with 2 filters + // -- lang facet for tag T2 + // -- tag facet for lang cpp & java + // -- repository for (cpp || java) & T2 + result = index.search(new RuleQuery() + .setLanguages(ImmutableList.of("cpp", "java")) + .setTags(ImmutableList.of("T2")) + , new SearchOptions().addFacets(asList("languages", "repositories", "tags"))); + assertThat(result.getIds()).hasSize(2); + assertThat(result.getFacets().getAll()).hasSize(3); + assertThat(result.getFacets().get(FACET_LANGUAGES).keySet()).containsOnly("cpp", "java"); + assertThat(result.getFacets().get(FACET_REPOSITORIES).keySet()).containsOnly("foo", "xoo"); + assertThat(result.getFacets().get(FACET_TAGS).keySet()).containsOnly("T1", "T2", "T3"); + } + + @Test + public void sort_by_name() { + indexRules( + newDoc(RuleKey.of("java", "S001")).setName("abcd"), + newDoc(RuleKey.of("java", "S002")).setName("ABC"), + newDoc(RuleKey.of("java", "S003")).setName("FGH")); + + // ascending + RuleQuery query = new RuleQuery().setSortField(RuleIndexDefinition.FIELD_RULE_NAME); + SearchIdResult results = index.search(query, new SearchOptions()); + assertThat(results.getIds()).containsExactly(RuleKey.of("java", "S002"), RuleKey.of("java", "S001"), RuleKey.of("java", "S003")); + + // descending + query = new RuleQuery().setSortField(RuleIndexDefinition.FIELD_RULE_NAME).setAscendingSort(false); + results = index.search(query, new SearchOptions()); + assertThat(results.getIds()).containsExactly(RuleKey.of("java", "S003"),RuleKey.of("java", "S001"), RuleKey.of("java", "S002")); + } + + @Test + public void default_sort_is_by_updated_at_desc() { + indexRules( + newDoc(RuleKey.of("java", "S001")).setCreatedAt(1000L).setUpdatedAt(1000L), + newDoc(RuleKey.of("java", "S002")).setCreatedAt(1000L).setUpdatedAt(3000L), + newDoc(RuleKey.of("java", "S003")).setCreatedAt(1000L).setUpdatedAt(2000L)); + + SearchIdResult results = index.search(new RuleQuery(), new SearchOptions()); + assertThat(results.getIds()).containsExactly(RuleKey.of("java", "S002"), RuleKey.of("java", "S003"), RuleKey.of("java", "S001")); + } + + @Test + public void fail_sort_by_language() { + try { + // Sorting on a field not tagged as sortable + new RuleQuery().setSortField(RuleIndexDefinition.FIELD_RULE_LANGUAGE); + fail(); + } catch (IllegalStateException e) { + assertThat(e).hasMessage("Field 'lang' is not sortable"); + } + } + + @Test + public void paging() { + indexRules( + newDoc(RuleKey.of("java", "S001")), + newDoc(RuleKey.of("java", "S002")), + newDoc(RuleKey.of("java", "S003"))); + + // from 0 to 1 included + SearchOptions options = new SearchOptions(); + options.setOffset(0).setLimit(2); + SearchIdResult results = index.search(new RuleQuery(), options); + assertThat(results.getTotal()).isEqualTo(3); + assertThat(results.getIds()).hasSize(2); + + // from 0 to 9 included + options.setOffset(0).setLimit(10); + results = index.search(new RuleQuery(), options); + assertThat(results.getTotal()).isEqualTo(3); + assertThat(results.getIds()).hasSize(3); + + // from 2 to 11 included + options.setOffset(2).setLimit(10); + results = index.search(new RuleQuery(), options); + assertThat(results.getTotal()).isEqualTo(3); + assertThat(results.getIds()).hasSize(1); + + // from 2 to 11 included + options.setOffset(2).setLimit(0); + results = index.search(new RuleQuery(), options); + assertThat(results.getTotal()).isEqualTo(3); + assertThat(results.getIds()).hasSize(1); + } + + @Test + public void search_all_keys_by_query() { + indexRules( + newDoc(RuleKey.of("javascript", "X001")), + newDoc(RuleKey.of("cobol", "X001")), + newDoc(RuleKey.of("php", "S002"))); + + // key + assertThat(index.searchAll(new RuleQuery().setQueryText("X001"))).hasSize(2); + + // partial key does not match + assertThat(index.searchAll(new RuleQuery().setQueryText("X00"))).isEmpty(); + + // repo:key -> nice-to-have ! + assertThat(index.searchAll(new RuleQuery().setQueryText("javascript:X001"))).hasSize(1); + } + + private void indexRules(RuleDoc... rules) { + ruleIndexer.index(asList(rules).iterator()); + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexDefinitionTest.java b/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexDefinitionTest.java new file mode 100644 index 00000000000..502ece8cd97 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexDefinitionTest.java @@ -0,0 +1,48 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.rule.index; + +import org.junit.Test; +import org.sonar.api.config.Settings; +import org.sonar.server.es.IndexDefinition; +import org.sonar.server.es.NewIndex; + +import static org.assertj.core.api.Assertions.assertThat; + +public class RuleIndexDefinitionTest { + + IndexDefinition.IndexDefinitionContext underTest = new IndexDefinition.IndexDefinitionContext(); + + @Test + public void define() { + RuleIndexDefinition def = new RuleIndexDefinition(new Settings()); + def.define(underTest); + + assertThat(underTest.getIndices()).hasSize(1); + NewIndex ruleIndex = underTest.getIndices().get("rules"); + assertThat(ruleIndex).isNotNull(); + assertThat(ruleIndex.getTypes().keySet()).containsOnly("rule"); + + // no cluster by default + assertThat(ruleIndex.getSettings().get("index.number_of_shards")).isEqualTo(String.valueOf(NewIndex.DEFAULT_NUMBER_OF_SHARDS)); + assertThat(ruleIndex.getSettings().get("index.number_of_replicas")).isEqualTo("0"); + } + +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexMediumTest.java deleted file mode 100644 index 6281782fcb2..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexMediumTest.java +++ /dev/null @@ -1,935 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2016 SonarSource SA - * mailto:contact AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.rule.index; - -import com.google.common.base.Function; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import com.google.common.collect.Sets; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import javax.annotation.Nullable; -import org.apache.commons.lang.time.DateUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.sonar.api.rule.RuleKey; -import org.sonar.api.rule.RuleStatus; -import org.sonar.api.rule.Severity; -import org.sonar.db.DbSession; -import org.sonar.db.qualityprofile.ActiveRuleDto; -import org.sonar.db.qualityprofile.QualityProfileDto; -import org.sonar.db.rule.RuleDto; -import org.sonar.db.rule.RuleParamDto; -import org.sonar.db.rule.RuleTesting; -import org.sonar.server.db.DbClient; -import org.sonar.server.qualityprofile.ActiveRule; -import org.sonar.server.qualityprofile.QProfileTesting; -import org.sonar.server.rule.Rule; -import org.sonar.server.rule.db.RuleDao; -import org.sonar.server.search.FacetValue; -import org.sonar.server.search.QueryContext; -import org.sonar.server.search.Result; -import org.sonar.server.tester.ServerTester; -import org.sonar.server.tester.UserSessionRule; - -import static com.google.common.collect.Lists.newArrayList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class RuleIndexMediumTest { - - @ClassRule - public static ServerTester tester = new ServerTester(); - @org.junit.Rule - public UserSessionRule userSessionRule = UserSessionRule.forServerTester(tester); - - protected DbClient db; - - RuleDao dao; - RuleIndex index; - // IndexClient index; - DbSession dbSession; - - @Before - public void before() { - dao = tester.get(RuleDao.class); - index = tester.get(RuleIndex.class); - tester.clearDbAndIndexes(); - db = tester.get(DbClient.class); - // index = tester.get(IndexClient.class); - dbSession = tester.get(DbClient.class).openSession(false); - - } - - @After - public void after() { - if (dbSession != null) { - dbSession.close(); - } - } - - @Test - public void getByKey() { - RuleDto ruleDto = RuleTesting.newDto(RuleKey.of("javascript", "S001")); - dao.insert(dbSession, ruleDto); - dbSession.commit(); - - Rule rule = index.getByKey(RuleKey.of("javascript", "S001")); - - assertThat(rule.htmlDescription()).isEqualTo(ruleDto.getDescription()); - assertThat(rule.key()).isEqualTo(ruleDto.getKey()); - - assertThat(rule.debtRemediationFunction().type().name()) - .isEqualTo(ruleDto.getRemediationFunction()); - - assertThat(Sets.newHashSet(rule.tags())).isEqualTo(ruleDto.getTags()); - assertThat(Sets.newHashSet(rule.systemTags())).isEqualTo(ruleDto.getSystemTags()); - } - - @Test - public void getByKey_null_if_not_found() { - Rule rule = index.getNullableByKey(RuleKey.of("javascript", "unknown")); - - assertThat(rule).isNull(); - } - - @Test - public void global_facet_on_repositories_and_tags() { - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("php", "S001")) - .setSystemTags(ImmutableSet.of("sysTag"))) - .setTags(ImmutableSet.of()); - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("php", "S002")) - .setSystemTags(ImmutableSet.of())) - .setTags(ImmutableSet.of("tag1")); - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("javascript", "S002")) - .setTags(ImmutableSet.of("tag1", "tag2"))) - .setSystemTags(ImmutableSet.of()); - dbSession.commit(); - - // should not have any facet! - RuleQuery query = new RuleQuery(); - Result result = index.search(query, new QueryContext(userSessionRule)); - assertThat(result.getFacets()).isEmpty(); - - // should not have any facet on non matching query! - result = index.search(new RuleQuery().setQueryText("aeiou"), new QueryContext(userSessionRule).addFacets(Arrays.asList("repositories"))); - assertThat(result.getFacets()).isEmpty(); - - // Repositories Facet is preset - result = index.search(query, new QueryContext(userSessionRule).addFacets(Arrays.asList("repositories", "tags"))); - assertThat(result.getFacets()).isNotNull(); - assertThat(result.getFacets()).hasSize(2); - - // Verify the value of a given facet - Collection repoFacets = result.getFacetValues("repositories"); - assertThat(repoFacets).hasSize(2); - assertThat(Iterables.get(repoFacets, 0).getKey()).isEqualTo("php"); - assertThat(Iterables.get(repoFacets, 0).getValue()).isEqualTo(2); - assertThat(Iterables.get(repoFacets, 1).getKey()).isEqualTo("javascript"); - assertThat(Iterables.get(repoFacets, 1).getValue()).isEqualTo(1); - - // Check that tag facet has both Tags and SystemTags values - Collection tagFacet = result.getFacetValues("tags"); - assertThat(tagFacet).hasSize(3); - assertThat(Iterables.get(tagFacet, 0).getKey()).isEqualTo("tag1"); - assertThat(Iterables.get(tagFacet, 0).getValue()).isEqualTo(2); - } - - @Test - public void return_all_doc_fields_by_default() { - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("javascript", "S001"))); - dbSession.commit(); - - QueryContext options = new QueryContext(userSessionRule).setFieldsToReturn(null); - Result results = index.search(new RuleQuery(), options); - assertThat(results.getHits()).hasSize(1); - Rule hit = Iterables.getFirst(results.getHits(), null); - assertThat(hit.key()).isNotNull(); - assertThat(hit.htmlDescription()).isNotNull(); - assertThat(hit.name()).isNotNull(); - - options = new QueryContext(userSessionRule).setFieldsToReturn(Collections.emptyList()); - results = index.search(new RuleQuery(), options); - assertThat(results.getHits()).hasSize(1); - hit = Iterables.getFirst(results.getHits(), null); - assertThat(hit.key()).isNotNull(); - assertThat(hit.htmlDescription()).isNotNull(); - assertThat(hit.name()).isNotNull(); - } - - @Test - public void select_doc_fields_to_return() { - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("javascript", "S001"))); - dbSession.commit(); - - QueryContext options = new QueryContext(userSessionRule); - options.addFieldsToReturn(RuleNormalizer.RuleField.LANGUAGE.field(), RuleNormalizer.RuleField.STATUS.field()); - Result results = index.search(new RuleQuery(), options); - assertThat(results.getHits()).hasSize(1); - - Rule hit = Iterables.getFirst(results.getHits(), null); - assertThat(hit.language()).isEqualTo("js"); - assertThat(hit.status()).isEqualTo(RuleStatus.READY); - - try { - hit.htmlDescription(); - fail(); - } catch (IllegalStateException e) { - assertThat(e).hasMessage("Field htmlDesc not specified in query options"); - } - } - - @Test - public void search_name_by_query() { - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("javascript", "S001")) - .setName("testing the partial match and matching of rule")); - dbSession.commit(); - - // substring - RuleQuery query = new RuleQuery().setQueryText("test"); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).hasSize(1); - - // substring - query = new RuleQuery().setQueryText("partial match"); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).hasSize(1); - - // case-insensitive - query = new RuleQuery().setQueryText("TESTING"); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).hasSize(1); - - // not found - query = new RuleQuery().setQueryText("not present"); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).isEmpty(); - } - - @Test - public void search_key_by_query() { - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("javascript", "X001"))); - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("cobol", "X001"))); - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("php", "S002"))); - dbSession.commit(); - - // key - RuleQuery query = new RuleQuery().setQueryText("X001"); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).hasSize(2); - - // partial key does not match - query = new RuleQuery().setQueryText("X00"); - // TODO fix non-partial match for Key search - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).isEmpty(); - - // repo:key -> nice-to-have ! - query = new RuleQuery().setQueryText("javascript:X001"); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).hasSize(1); - } - - @Test - public void filter_by_key() { - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("javascript", "X001"))); - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("cobol", "X001"))); - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("php", "S002"))); - dbSession.commit(); - - // key - RuleQuery query = new RuleQuery().setKey(RuleKey.of("javascript", "X001").toString()); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).hasSize(1); - - // partial key does not match - query = new RuleQuery().setKey("X001"); - // TODO fix non-partial match for Key search - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).isEmpty(); - } - - @Test - public void search_all_rules() { - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("javascript", "S001"))); - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("java", "S002"))); - dbSession.commit(); - - Result results = index.search(new RuleQuery(), new QueryContext(userSessionRule)); - - assertThat(results.getTotal()).isEqualTo(2); - assertThat(results.getHits()).hasSize(2); - } - - @Test - public void scroll_all_rules() { - int max = 100; - for (int i = 0; i < max; i++) { - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("java", "scroll_" + i))); - } - dbSession.commit(); - - Result results = index.search(new RuleQuery(), new QueryContext(userSessionRule).setScroll(true)); - - assertThat(results.getTotal()).isEqualTo(max); - assertThat(results.getHits()).hasSize(0); - - Iterator it = results.scroll(); - int count = 0; - while (it.hasNext()) { - count++; - it.next(); - } - assertThat(count).isEqualTo(max); - - } - - @Test - public void search_by_any_of_repositories() { - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("findbugs", "S001"))); - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("pmd", "S002"))); - dbSession.commit(); - - RuleQuery query = new RuleQuery().setRepositories(Arrays.asList("checkstyle", "pmd")); - Result results = index.search(query, new QueryContext(userSessionRule)); - assertThat(results.getHits()).hasSize(1); - assertThat(Iterables.getFirst(results.getHits(), null).key().rule()).isEqualTo("S002"); - - // no results - query = new RuleQuery().setRepositories(Arrays.asList("checkstyle")); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).isEmpty(); - - // empty list => no filter - query = new RuleQuery().setRepositories(Collections.emptyList()); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).hasSize(2); - } - - @Test - public void search_by_any_of_languages() { - dao.insert(dbSession, - RuleTesting.newDto(RuleKey.of("java", "S001")).setLanguage("java"), - RuleTesting.newDto(RuleKey.of("javascript", "S002")).setLanguage("js")); - dbSession.commit(); - - RuleQuery query = new RuleQuery().setLanguages(Arrays.asList("cobol", "js")); - Result results = index.search(query, new QueryContext(userSessionRule)); - - assertThat(results.getHits()).hasSize(1); - assertThat(Iterables.getFirst(results.getHits(), null).key().rule()).isEqualTo("S002"); - - // no results - query = new RuleQuery().setLanguages(Arrays.asList("cpp")); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).isEmpty(); - - // empty list => no filter - query = new RuleQuery().setLanguages(Collections.emptyList()); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).hasSize(2); - - // null list => no filter - query = new RuleQuery().setLanguages(null); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).hasSize(2); - } - - @Test - public void search_by_any_of_severities() { - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("java", "S001")).setSeverity(Severity.BLOCKER)); - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("java", "S002")).setSeverity(Severity.INFO)); - dbSession.commit(); - - RuleQuery query = new RuleQuery().setSeverities(Arrays.asList(Severity.INFO, Severity.MINOR)); - Result results = index.search(query, new QueryContext(userSessionRule)); - assertThat(results.getHits()).hasSize(1); - assertThat(Iterables.getFirst(results.getHits(), null).key().rule()).isEqualTo("S002"); - - // no results - query = new RuleQuery().setSeverities(Arrays.asList(Severity.MINOR)); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).isEmpty(); - - // empty list => no filter - query = new RuleQuery().setSeverities(Collections.emptyList()); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).hasSize(2); - - // null list => no filter - query = new RuleQuery().setSeverities(null); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).hasSize(2); - } - - @Test - public void search_by_any_of_statuses() { - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("java", "S001")).setStatus(RuleStatus.BETA)); - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("java", "S002")).setStatus(RuleStatus.READY)); - dbSession.commit(); - - RuleQuery query = new RuleQuery().setStatuses(Arrays.asList(RuleStatus.DEPRECATED, RuleStatus.READY)); - Result results = index.search(query, new QueryContext(userSessionRule)); - assertThat(results.getHits()).hasSize(1); - assertThat(Iterables.getFirst(results.getHits(), null).key().rule()).isEqualTo("S002"); - - // no results - query = new RuleQuery().setStatuses(Arrays.asList(RuleStatus.DEPRECATED)); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).isEmpty(); - - // empty list => no filter - query = new RuleQuery().setStatuses(Collections.emptyList()); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).hasSize(2); - - // null list => no filter - query = new RuleQuery().setStatuses(null); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).hasSize(2); - } - - @Test - public void sort_by_name() { - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("java", "S001")).setName("abcd")); - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("java", "S002")).setName("ABC")); - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("java", "S003")).setName("FGH")); - dbSession.commit(); - - // ascending - RuleQuery query = new RuleQuery().setSortField(RuleNormalizer.RuleField.NAME); - Result results = index.search(query, new QueryContext(userSessionRule)); - assertThat(results.getHits()).hasSize(3); - assertThat(Iterables.getFirst(results.getHits(), null).key().rule()).isEqualTo("S002"); - assertThat(Iterables.getLast(results.getHits(), null).key().rule()).isEqualTo("S003"); - - // descending - query = new RuleQuery().setSortField(RuleNormalizer.RuleField.NAME).setAscendingSort(false); - results = index.search(query, new QueryContext(userSessionRule)); - assertThat(results.getHits()).hasSize(3); - assertThat(Iterables.getFirst(results.getHits(), null).key().rule()).isEqualTo("S003"); - assertThat(Iterables.getLast(results.getHits(), null).key().rule()).isEqualTo("S002"); - } - - @Test - public void fail_sort_by_language() { - try { - // Sorting on a field not tagged as sortable - new RuleQuery().setSortField(RuleNormalizer.RuleField.LANGUAGE); - fail(); - } catch (IllegalStateException e) { - assertThat(e).hasMessage("Field 'lang' is not sortable"); - } - } - - @Test - public void search_by_profile() { - QualityProfileDto qualityProfileDto1 = QProfileTesting.newXooP1(); - QualityProfileDto qualityProfileDto2 = QProfileTesting.newXooP2(); - db.qualityProfileDao().insert(dbSession, qualityProfileDto1, qualityProfileDto2); - - RuleDto rule1 = RuleTesting.newXooX1(); - RuleDto rule2 = RuleTesting.newXooX2(); - RuleDto rule3 = RuleTesting.newXooX3(); - dao.insert(dbSession, rule1, rule2, rule3); - - db.activeRuleDao().insert( - dbSession, - ActiveRuleDto.createFor(qualityProfileDto1, rule1).setSeverity("BLOCKER"), - ActiveRuleDto.createFor(qualityProfileDto2, rule1).setSeverity("BLOCKER"), - ActiveRuleDto.createFor(qualityProfileDto1, rule2).setSeverity("BLOCKER")); - dbSession.commit(); - dbSession.clearCache(); - - // 1. get all active rules. - Result result = index.search(new RuleQuery().setActivation(true), - new QueryContext(userSessionRule)); - assertThat(result.getHits()).hasSize(2); - - // 2. get all inactive rules. - result = index.search(new RuleQuery().setActivation(false), - new QueryContext(userSessionRule)); - assertThat(result.getHits()).hasSize(1); - assertThat(result.getHits().get(0).name()).isEqualTo(rule3.getName()); - - // 3. get all rules not active on profile - index.search(new RuleQuery().setActivation(false).setQProfileKey(qualityProfileDto2.getKey()), - new QueryContext(userSessionRule)); - // TODO - assertThat(result.getHits()).hasSize(1); - - // 4. get all active rules on profile - result = index.search(new RuleQuery().setActivation(true) - .setQProfileKey(qualityProfileDto2.getKey()), - new QueryContext(userSessionRule)); - assertThat(result.getHits()).hasSize(1); - assertThat(result.getHits().get(0).name()).isEqualTo(rule1.getName()); - - } - - @Test - public void search_by_profile_and_inheritance() { - QualityProfileDto qualityProfileDto1 = QProfileTesting.newXooP1(); - QualityProfileDto qualityProfileDto2 = QProfileTesting.newXooP2().setParentKee(QProfileTesting.XOO_P1_KEY); - db.qualityProfileDao().insert(dbSession, qualityProfileDto1, qualityProfileDto2); - - RuleDto rule1 = RuleTesting.newDto(RuleKey.of("xoo", "S001")); - RuleDto rule2 = RuleTesting.newDto(RuleKey.of("xoo", "S002")); - RuleDto rule3 = RuleTesting.newDto(RuleKey.of("xoo", "S003")); - RuleDto rule4 = RuleTesting.newDto(RuleKey.of("xoo", "S004")); - dao.insert(dbSession, rule1, rule2, rule3, rule4); - - db.activeRuleDao().insert( - dbSession, - ActiveRuleDto.createFor(qualityProfileDto1, rule1) - .setSeverity("BLOCKER"), - ActiveRuleDto.createFor(qualityProfileDto1, rule2) - .setSeverity("BLOCKER"), - ActiveRuleDto.createFor(qualityProfileDto1, rule3) - .setSeverity("BLOCKER"), - - ActiveRuleDto.createFor(qualityProfileDto2, rule1) - .setSeverity("MINOR") - .setInheritance(ActiveRule.Inheritance.INHERITED.name()), - ActiveRuleDto.createFor(qualityProfileDto2, rule2) - .setSeverity("BLOCKER") - .setInheritance(ActiveRule.Inheritance.OVERRIDES.name()), - ActiveRuleDto.createFor(qualityProfileDto2, rule3) - .setSeverity("BLOCKER") - .setInheritance(ActiveRule.Inheritance.INHERITED.name()) - ); - - dbSession.commit(); - - // 0. get all rules - Result result = index.search(new RuleQuery(), - new QueryContext(userSessionRule)); - assertThat(result.getHits()).hasSize(4); - - // 1. get all active rules - result = index.search(new RuleQuery().setActivation(true), - new QueryContext(userSessionRule)); - assertThat(result.getHits()).hasSize(3); - - // 2. get all inactive rules. - result = index.search(new RuleQuery().setActivation(false), - new QueryContext(userSessionRule)); - assertThat(result.getHits()).hasSize(1); - assertThat(result.getHits().get(0).name()).isEqualTo(rule4.getName()); - - // 3. get Inherited Rules on profile1 - result = index.search(new RuleQuery().setActivation(true) - .setQProfileKey(qualityProfileDto1.getKey()) - .setInheritance(ImmutableSet.of(ActiveRule.Inheritance.INHERITED.name())), - new QueryContext(userSessionRule) - ); - assertThat(result.getHits()).hasSize(0); - - // 4. get Inherited Rules on profile2 - result = index.search(new RuleQuery().setActivation(true) - .setQProfileKey(qualityProfileDto2.getKey()) - .setInheritance(ImmutableSet.of(ActiveRule.Inheritance.INHERITED.name())), - new QueryContext(userSessionRule) - ); - assertThat(result.getHits()).hasSize(2); - - // 5. get Overridden Rules on profile1 - result = index.search(new RuleQuery().setActivation(true) - .setQProfileKey(qualityProfileDto1.getKey()) - .setInheritance(ImmutableSet.of(ActiveRule.Inheritance.OVERRIDES.name())), - new QueryContext(userSessionRule) - ); - assertThat(result.getHits()).hasSize(0); - - // 6. get Overridden Rules on profile2 - result = index.search(new RuleQuery().setActivation(true) - .setQProfileKey(qualityProfileDto2.getKey()) - .setInheritance(ImmutableSet.of(ActiveRule.Inheritance.OVERRIDES.name())), - new QueryContext(userSessionRule) - ); - assertThat(result.getHits()).hasSize(1); - - // 7. get Inherited AND Overridden Rules on profile1 - result = index.search(new RuleQuery().setActivation(true) - .setQProfileKey(qualityProfileDto1.getKey()) - .setInheritance(ImmutableSet.of( - ActiveRule.Inheritance.INHERITED.name(), ActiveRule.Inheritance.OVERRIDES.name())), - new QueryContext(userSessionRule) - ); - assertThat(result.getHits()).hasSize(0); - - // 8. get Inherited AND Overridden Rules on profile2 - result = index.search(new RuleQuery().setActivation(true) - .setQProfileKey(qualityProfileDto2.getKey()) - .setInheritance(ImmutableSet.of( - ActiveRule.Inheritance.INHERITED.name(), ActiveRule.Inheritance.OVERRIDES.name())), - new QueryContext(userSessionRule) - ); - assertThat(result.getHits()).hasSize(3); - } - - @Test - public void search_by_profile_and_active_severity() { - QualityProfileDto qualityProfileDto1 = QProfileTesting.newXooP1(); - QualityProfileDto qualityProfileDto2 = QProfileTesting.newXooP2(); - db.qualityProfileDao().insert(dbSession, qualityProfileDto1, qualityProfileDto2); - - RuleDto rule1 = RuleTesting.newXooX1().setSeverity("MAJOR"); - RuleDto rule2 = RuleTesting.newXooX2().setSeverity("MINOR"); - RuleDto rule3 = RuleTesting.newXooX3().setSeverity("INFO"); - dao.insert(dbSession, rule1, rule2, rule3); - - db.activeRuleDao().insert( - dbSession, - ActiveRuleDto.createFor(qualityProfileDto1, rule1).setSeverity("BLOCKER"), - ActiveRuleDto.createFor(qualityProfileDto2, rule1).setSeverity("BLOCKER"), - ActiveRuleDto.createFor(qualityProfileDto1, rule2).setSeverity("CRITICAL")); - dbSession.commit(); - dbSession.clearCache(); - - // 1. get all active rules. - Result result = index.search(new RuleQuery().setActivation(true).setQProfileKey(qualityProfileDto1.getKey()), - new QueryContext(userSessionRule)); - assertThat(result.getHits()).hasSize(2); - - // 2. get rules with active severity critical. - result = index.search(new RuleQuery().setActivation(true).setQProfileKey(qualityProfileDto1.getKey()).setActiveSeverities(Arrays.asList("CRITICAL")), - new QueryContext(userSessionRule).addFacets(Arrays.asList(RuleIndex.FACET_ACTIVE_SEVERITIES))); - assertThat(result.getHits()).hasSize(1); - assertThat(result.getHits().get(0).name()).isEqualTo(rule2.getName()); - // check stickyness of active severity facet - assertThat(result.getFacetValues(RuleIndex.FACET_ACTIVE_SEVERITIES)).containsOnly(new FacetValue("BLOCKER", 1), new FacetValue("CRITICAL", 1)); - - // 3. count activation severities of all active rules - result = index.search(new RuleQuery(), - new QueryContext(userSessionRule).addFacets(Arrays.asList(RuleIndex.FACET_ACTIVE_SEVERITIES))); - assertThat(result.getHits()).hasSize(3); - assertThat(result.getFacetValues(RuleIndex.FACET_ACTIVE_SEVERITIES)).containsOnly(new FacetValue("BLOCKER", 2), new FacetValue("CRITICAL", 1)); - } - - @Test - public void complex_param_value() { - String value = "//expression[primary/qualifiedIdentifier[count(IDENTIFIER) = 2]/IDENTIFIER[2]/@tokenValue = 'firstOf' and primary/identifierSuffix/arguments/expression[not(primary) or primary[not(qualifiedIdentifier) or identifierSuffix]]]"; - - QualityProfileDto profile = QProfileTesting.newXooP1(); - db.qualityProfileDao().insert(dbSession, profile); - - RuleDto rule = RuleTesting.newXooX1(); - dao.insert(dbSession, rule); - - RuleParamDto param = RuleParamDto.createFor(rule) - .setName("testing") - .setType("STRING") - .setDefaultValue(value); - dao.insertRuleParam(dbSession, rule, param); - - dbSession.commit(); - - assertThat(index.getByKey(rule.getKey()).params()).hasSize(1); - assertThat(index.getByKey(rule.getKey()).params().get(0).defaultValue()).isEqualTo(value); - } - - @Test - public void search_by_tag() { - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("java", "S001")).setTags(ImmutableSet.of("tag1"))); - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("java", "S002")).setTags(ImmutableSet.of("tag2"))); - dbSession.commit(); - - // find all - RuleQuery query = new RuleQuery(); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).hasSize(2); - - // tag1 in query - query = new RuleQuery().setQueryText("tag1"); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).hasSize(1); - assertThat(Iterables.getFirst(index.search(query, new QueryContext(userSessionRule)).getHits(), null).tags()).containsExactly("tag1"); - - // tag1 and tag2 in query - query = new RuleQuery().setQueryText("tag1 tag2"); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).hasSize(2); - - // tag2 in filter - query = new RuleQuery().setTags(ImmutableSet.of("tag2")); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).hasSize(1); - assertThat(Iterables.getFirst(index.search(query, new QueryContext(userSessionRule)).getHits(), null).tags()).containsExactly("tag2"); - - // tag2 in filter and tag1 tag2 in query - query = new RuleQuery().setTags(ImmutableSet.of("tag2")).setQueryText("tag1"); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).hasSize(0); - - // tag2 in filter and tag1 in query - query = new RuleQuery().setTags(ImmutableSet.of("tag2")).setQueryText("tag1 tag2"); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).hasSize(1); - assertThat(Iterables.getFirst(index.search(query, new QueryContext(userSessionRule)).getHits(), null).tags()).containsExactly("tag2"); - - // null list => no filter - query = new RuleQuery().setTags(Collections.emptySet()); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).hasSize(2); - - // null list => no filter - query = new RuleQuery().setTags(null); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).hasSize(2); - } - - @Test - public void search_by_is_template() { - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("java", "S001")).setIsTemplate(false)); - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("java", "S002")).setIsTemplate(true)); - dbSession.commit(); - - // find all - RuleQuery query = new RuleQuery(); - Result results = index.search(query, new QueryContext(userSessionRule)); - assertThat(results.getHits()).hasSize(2); - - // Only template - query = new RuleQuery().setIsTemplate(true); - results = index.search(query, new QueryContext(userSessionRule)); - assertThat(results.getHits()).hasSize(1); - assertThat(Iterables.getFirst(results.getHits(), null).key().rule()).isEqualTo("S002"); - assertThat(Iterables.getFirst(results.getHits(), null).isTemplate()).isTrue(); - - // Only not template - query = new RuleQuery().setIsTemplate(false); - results = index.search(query, new QueryContext(userSessionRule)); - assertThat(results.getHits()).hasSize(1); - assertThat(Iterables.getFirst(results.getHits(), null).isTemplate()).isFalse(); - assertThat(Iterables.getFirst(results.getHits(), null).key().rule()).isEqualTo("S001"); - - // null => no filter - query = new RuleQuery().setIsTemplate(null); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).hasSize(2); - } - - @Test - public void search_by_template_key() { - RuleDto templateRule = RuleTesting.newDto(RuleKey.of("java", "S001")).setIsTemplate(true); - dao.insert(dbSession, templateRule); - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("java", "S001_MY_CUSTOM")).setTemplateId(templateRule.getId())); - dbSession.commit(); - - // find all - RuleQuery query = new RuleQuery(); - Result results = index.search(query, new QueryContext(userSessionRule)); - assertThat(results.getHits()).hasSize(2); - - // Only custom rule - query = new RuleQuery().setTemplateKey("java:S001"); - results = index.search(query, new QueryContext(userSessionRule)); - assertThat(results.getHits()).hasSize(1); - assertThat(Iterables.getFirst(results.getHits(), null).key().rule()).isEqualTo("S001_MY_CUSTOM"); - assertThat(Iterables.getFirst(results.getHits(), null).templateKey()).isEqualTo(RuleKey.of("java", "S001")); - - // null => no filter - query = new RuleQuery().setTemplateKey(null); - assertThat(index.search(query, new QueryContext(userSessionRule)).getHits()).hasSize(2); - } - - @Test - public void search_by_template_key_with_params() { - RuleDto templateRule = RuleTesting.newDto(RuleKey.of("java", "S001")).setIsTemplate(true); - RuleParamDto ruleParamDto = RuleParamDto.createFor(templateRule).setName("regex").setType("STRING").setDescription("Reg ex").setDefaultValue(".*"); - dao.insert(dbSession, templateRule); - dao.insertRuleParam(dbSession, templateRule, ruleParamDto); - - RuleDto customRule = RuleTesting.newDto(RuleKey.of("java", "S001_MY_CUSTOM")).setTemplateId(templateRule.getId()); - RuleParamDto customRuleParam = RuleParamDto.createFor(customRule).setName("regex").setType("STRING").setDescription("Reg ex").setDefaultValue("a.*"); - dao.insert(dbSession, customRule); - dao.insertRuleParam(dbSession, customRule, customRuleParam); - dbSession.commit(); - - // find all - RuleQuery query = new RuleQuery(); - Result results = index.search(query, new QueryContext(userSessionRule)); - assertThat(results.getHits()).hasSize(2); - - // get params - assertThat(index.getByKey(templateRule.getKey()).params()).hasSize(1); - assertThat(index.getByKey(customRule.getKey()).params()).hasSize(1); - } - - @Test - public void show_custom_rule() { - RuleDto templateRule = RuleTesting.newDto(RuleKey.of("java", "S001")).setIsTemplate(true); - dao.insert(dbSession, templateRule); - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("java", "S001_MY_CUSTOM")).setTemplateId(templateRule.getId())); - dbSession.commit(); - - // find all - RuleQuery query = new RuleQuery(); - Result results = index.search(query, new QueryContext(userSessionRule)); - assertThat(results.getHits()).hasSize(2); - - // find custom rule - assertThat(index.getByKey(RuleKey.of("java", "S001_MY_CUSTOM")).templateKey()).isEqualTo(RuleKey.of("java", "S001")); - } - - @Test - public void paging() { - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("java", "S001"))); - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("java", "S002"))); - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("java", "S003"))); - dbSession.commit(); - - // from 0 to 1 included - QueryContext options = new QueryContext(userSessionRule); - options.setOffset(0).setLimit(2); - Result results = index.search(new RuleQuery(), options); - assertThat(results.getTotal()).isEqualTo(3); - assertThat(results.getHits()).hasSize(2); - - // from 0 to 9 included - options.setOffset(0).setLimit(10); - results = index.search(new RuleQuery(), options); - assertThat(results.getTotal()).isEqualTo(3); - assertThat(results.getHits()).hasSize(3); - - // from 2 to 11 included - options.setOffset(2).setLimit(10); - results = index.search(new RuleQuery(), options); - assertThat(results.getTotal()).isEqualTo(3); - assertThat(results.getHits()).hasSize(1); - - // from 2 to 11 included - options.setOffset(2).setLimit(0); - results = index.search(new RuleQuery(), options); - assertThat(results.getTotal()).isEqualTo(3); - assertThat(results.getHits()).hasSize(0); - } - - @Test - public void available_since() throws InterruptedException { - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("java", "S001"))); - dbSession.commit(); - - Date since = new Date(); - dao.insert(dbSession, RuleTesting.newDto(RuleKey.of("java", "S002"))); - dbSession.commit(); - - // 0. find all rules; - assertThat(index.search(new RuleQuery(), new QueryContext(userSessionRule)).getHits()).hasSize(2); - - // 1. find all rules available since a date; - RuleQuery availableSinceQuery = new RuleQuery() - .setAvailableSince(since); - List hits = index.search(availableSinceQuery, new QueryContext(userSessionRule)).getHits(); - assertThat(hits).hasSize(1); - assertThat(hits.get(0).key()).isEqualTo(RuleKey.of("java", "S002")); - - // 2. find no new rules since tomorrow. - RuleQuery availableSinceNowQuery = new RuleQuery() - .setAvailableSince(DateUtils.addDays(since, 1)); - assertThat(index.search(availableSinceNowQuery, new QueryContext(userSessionRule)).getHits()).hasSize(0); - } - - @Test - public void scroll_byIds() { - Set ids = new HashSet<>(); - for (int i = 0; i < 150; i++) { - RuleDto rule = RuleTesting.newDto(RuleKey.of("scroll", "r_" + i)); - dao.insert(dbSession, rule); - dbSession.commit(); - ids.add(rule.getId()); - } - List rules = index.getByIds(ids); - assertThat(rules).hasSize(ids.size()); - } - - @Test - public void search_protected_chars() { - String nameWithProtectedChars = "ja#va&sc\"r:ipt"; - - RuleDto ruleDto = RuleTesting.newXooX1().setName(nameWithProtectedChars); - dao.insert(dbSession, ruleDto); - dbSession.commit(); - - Rule rule = index.getByKey(RuleTesting.XOO_X1); - assertThat(rule.name()).isEqualTo(nameWithProtectedChars); - - RuleQuery protectedCharsQuery = new RuleQuery().setQueryText(nameWithProtectedChars); - List results = index.search(protectedCharsQuery, new QueryContext(userSessionRule)).getHits(); - assertThat(results).hasSize(1); - assertThat(results.get(0).key()).isEqualTo(RuleTesting.XOO_X1); - } - - @Test - public void sticky_facets() { - - dao.insert(dbSession, - RuleTesting.newDto(RuleKey.of("xoo", "S001")).setLanguage("java").setTags(ImmutableSet.of()), - RuleTesting.newDto(RuleKey.of("xoo", "S002")).setLanguage("java").setTags(ImmutableSet.of()), - RuleTesting.newDto(RuleKey.of("xoo", "S003")).setLanguage("java").setTags(ImmutableSet.of("T1", "T2")), - RuleTesting.newDto(RuleKey.of("xoo", "S011")).setLanguage("cobol").setTags(ImmutableSet.of()), - RuleTesting.newDto(RuleKey.of("xoo", "S012")).setLanguage("cobol").setTags(ImmutableSet.of()), - RuleTesting.newDto(RuleKey.of("foo", "S013")).setLanguage("cobol").setTags(ImmutableSet.of("T3", "T4")), - RuleTesting.newDto(RuleKey.of("foo", "S111")).setLanguage("cpp").setTags(ImmutableSet.of()), - RuleTesting.newDto(RuleKey.of("foo", "S112")).setLanguage("cpp").setTags(ImmutableSet.of()), - RuleTesting.newDto(RuleKey.of("foo", "S113")).setLanguage("cpp").setTags(ImmutableSet.of("T2", "T3"))); - dbSession.commit(); - - // 0 assert Base - assertThat(index.countAll()).isEqualTo(9); - assertThat(index.search(new RuleQuery(), new QueryContext(userSessionRule)).getHits()).hasSize(9); - - // 1 Facet with no filters at all - Map> facets = index.search(new RuleQuery(), new QueryContext(userSessionRule).addFacets(Arrays.asList("languages", "repositories", "tags"))) - .getFacets(); - assertThat(facets.keySet()).hasSize(3); - assertThat(facets.get(RuleIndex.FACET_LANGUAGES)).extracting("key").containsOnly("cpp", "java", "cobol"); - assertThat(facets.get(RuleIndex.FACET_REPOSITORIES)).extracting("key").containsOnly("xoo", "foo"); - assertThat(facets.get(RuleIndex.FACET_TAGS)).extracting("key").containsOnly("systag1", "systag2", "T1", "T2", "T3", "T4"); - - // 2 Facet with a language filter - // -- lang facet should still have all language - Result result = index.search(new RuleQuery() - .setLanguages(ImmutableList.of("cpp")) - , new QueryContext(userSessionRule).addFacets(Arrays.asList("languages", "repositories", "tags"))); - assertThat(result.getHits()).hasSize(3); - assertThat(result.getFacets()).hasSize(3); - assertThat(result.getFacets().get(RuleIndex.FACET_LANGUAGES)).extracting("key").containsOnly("cpp", "java", "cobol"); - - // 3 facet with 2 filters - // -- lang facet for tag T2 - // -- tag facet for lang cpp - // -- repository for cpp & T2 - result = index.search(new RuleQuery() - .setLanguages(ImmutableList.of("cpp")) - .setTags(ImmutableList.of("T2")) - , new QueryContext(userSessionRule).addFacets(Arrays.asList("languages", "repositories", "tags"))); - assertThat(result.getHits()).hasSize(1); - assertThat(result.getFacets().keySet()).hasSize(3); - assertThat(result.getFacets().get(RuleIndex.FACET_LANGUAGES)).extracting("key").containsOnly("cpp", "java"); - assertThat(result.getFacets().get(RuleIndex.FACET_REPOSITORIES)).extracting("key").containsOnly("foo"); - assertThat(result.getFacets().get(RuleIndex.FACET_TAGS)).extracting("key").containsOnly("systag1", "systag2", "T2", "T3"); - - // 4 facet with 2 filters - // -- lang facet for tag T2 - // -- tag facet for lang cpp & java - // -- repository for (cpp || java) & T2 - result = index.search(new RuleQuery() - .setLanguages(ImmutableList.of("cpp", "java")) - .setTags(ImmutableList.of("T2")) - , new QueryContext(userSessionRule).addFacets(Arrays.asList("languages", "repositories", "tags"))); - assertThat(result.getHits()).hasSize(2); - assertThat(result.getFacets().keySet()).hasSize(3); - assertThat(result.getFacets().get(RuleIndex.FACET_LANGUAGES)).extracting("key").containsOnly("cpp", "java"); - assertThat(result.getFacets().get(RuleIndex.FACET_REPOSITORIES)).extracting("key").containsOnly("foo", "xoo"); - assertThat(result.getFacets().get(RuleIndex.FACET_TAGS)).extracting("key").containsOnly("systag1", "systag2", "T1", "T2", "T3"); - } - - private static List ruleKeys(List rules) { - return newArrayList(Iterables.transform(rules, new Function() { - @Override - public String apply(@Nullable Rule input) { - return input != null ? input.key().rule() : null; - } - })); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java new file mode 100644 index 00000000000..e1914d17251 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java @@ -0,0 +1,109 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.rule.index; + +import com.google.common.collect.Iterators; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.sonar.api.config.Settings; +import org.sonar.api.rule.RuleKey; +import org.sonar.api.rule.RuleStatus; +import org.sonar.api.utils.System2; +import org.sonar.db.DbClient; +import org.sonar.db.DbTester; +import org.sonar.db.rule.RuleDto; +import org.sonar.db.rule.RuleTesting; +import org.sonar.server.es.EsTester; +import org.sonar.test.DbTests; + +import static org.assertj.core.api.Assertions.assertThat; + +@Category(DbTests.class) +public class RuleIndexerTest { + + @ClassRule + public static EsTester esTester = new EsTester().addDefinitions(new RuleIndexDefinition(new Settings())); + + @Rule + public DbTester dbTester = DbTester.create(System2.INSTANCE); + + @Before + public void setUp() { + esTester.truncateIndices(); + } + + @Test + public void index_nothing() { + RuleIndexer indexer = createIndexer(); + indexer.index(Iterators.emptyIterator()); + assertThat(esTester.countDocuments(RuleIndexDefinition.INDEX, RuleIndexDefinition.TYPE_RULE)).isEqualTo(0L); + } + + @Test + public void index_nothing_if_disabled() { + dbTester.prepareDbUnit(getClass(), "index.xml"); + + createIndexer().setEnabled(false).index(); + + assertThat(esTester.countDocuments(RuleIndexDefinition.INDEX, RuleIndexDefinition.TYPE_RULE)).isZero(); + } + + @Test + public void index() { + dbTester.prepareDbUnit(getClass(), "index.xml"); + + RuleIndexer indexer = createIndexer(); + indexer.index(); + + assertThat(esTester.countDocuments(RuleIndexDefinition.INDEX, RuleIndexDefinition.TYPE_RULE)).isEqualTo(1); + } + + @Test + public void removed_rule_is_removed_from_index() { + RuleIndexer indexer = createIndexer(); + + // Create and Index rule + RuleDto ruleDto = RuleTesting.newDto(RuleKey.of("xoo", "S001")) + .setStatus(RuleStatus.READY) + .setUpdatedAtInMs(1000L); + dbTester.getDbClient().ruleDao().insert(dbTester.getSession(), ruleDto); + dbTester.getSession().commit(); + indexer.index(); + + // Remove rule + ruleDto.setStatus(RuleStatus.REMOVED); + ruleDto.setUpdatedAtInMs(2000L); + dbTester.getDbClient().ruleDao().update(dbTester.getSession(), ruleDto); + dbTester.getSession().commit(); + indexer.index(); + + assertThat(esTester.countDocuments(RuleIndexDefinition.INDEX, RuleIndexDefinition.TYPE_RULE)).isZero(); + } + + private RuleIndexer createIndexer() { + RuleIndexer indexer = new RuleIndexer(new DbClient(dbTester.database(), dbTester.myBatis()), esTester.client()); + indexer.setEnabled(true); + return indexer; + } + +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleResultSetIteratorTest.java b/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleResultSetIteratorTest.java new file mode 100644 index 00000000000..f6abb47f65b --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleResultSetIteratorTest.java @@ -0,0 +1,169 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.rule.index; + +import com.google.common.base.Function; +import com.google.common.collect.Maps; +import java.util.Map; +import javax.annotation.Nonnull; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.sonar.api.rule.RuleKey; +import org.sonar.api.rule.RuleStatus; +import org.sonar.api.rule.Severity; +import org.sonar.api.utils.System2; +import org.sonar.db.DbTester; +import org.sonar.test.DbTests; + +import static org.assertj.core.api.Assertions.assertThat; + +@Category(DbTests.class) +public class RuleResultSetIteratorTest { + + @Rule + public DbTester dbTester = DbTester.create(System2.INSTANCE); + + @Before + public void setUp() { + dbTester.truncateTables(); + } + + @Test + public void iterator_over_one_rule() { + dbTester.prepareDbUnit(getClass(), "one_rule.xml"); + RuleResultSetIterator it = RuleResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), 0L); + Map rulesByKey = rulesByKey(it); + it.close(); + + assertThat(rulesByKey).hasSize(1); + + RuleDoc rule = rulesByKey.get("S001"); + assertThat(rule.key()).isEqualTo(RuleKey.of("xoo", "S001")); + assertThat(rule.keyAsList()).containsOnly("xoo", "S001"); + assertThat(rule.ruleKey()).isEqualTo("S001"); + assertThat(rule.repository()).isEqualTo("xoo"); + assertThat(rule.internalKey()).isEqualTo("S1"); + assertThat(rule.name()).isEqualTo("Null Pointer"); + assertThat(rule.htmlDescription()).isEqualTo("S001 desc"); + assertThat(rule.language()).isEqualTo("xoo"); + assertThat(rule.severity()).isEqualTo(Severity.BLOCKER); + assertThat(rule.status()).isEqualTo(RuleStatus.READY); + assertThat(rule.isTemplate()).isFalse(); + assertThat(rule.allTags()).containsOnly("bug", "performance", "cwe"); + assertThat(rule.createdAtAsLong()).isEqualTo(1500000000000L); + assertThat(rule.updatedAtAtAsLong()).isEqualTo(1600000000000L); + } + + @Test + public void select_after_date() { + dbTester.prepareDbUnit(getClass(), "shared.xml"); + RuleResultSetIterator it = RuleResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), 1_900_000_000_000L); + + assertThat(it.hasNext()).isTrue(); + RuleDoc issue = it.next(); + assertThat(issue.key()).isEqualTo(RuleKey.of("xoo", "S002")); + + assertThat(it.hasNext()).isFalse(); + it.close(); + } + + @Test + public void iterator_over_rules() { + dbTester.prepareDbUnit(getClass(), "shared.xml"); + + RuleResultSetIterator it = RuleResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), 0L); + Map rulesByKey = rulesByKey(it); + it.close(); + + assertThat(rulesByKey).hasSize(2); + + RuleDoc rule = rulesByKey.get("S001"); + assertThat(rule.key()).isEqualTo(RuleKey.of("xoo", "S001")); + assertThat(rule.keyAsList()).containsOnly("xoo", "S001"); + assertThat(rule.ruleKey()).isEqualTo("S001"); + assertThat(rule.repository()).isEqualTo("xoo"); + assertThat(rule.internalKey()).isEqualTo("S1"); + assertThat(rule.name()).isEqualTo("Null Pointer"); + assertThat(rule.htmlDescription()).isEqualTo("S001 desc"); + assertThat(rule.language()).isEqualTo("xoo"); + assertThat(rule.severity()).isEqualTo(Severity.BLOCKER); + assertThat(rule.status()).isEqualTo(RuleStatus.READY); + assertThat(rule.isTemplate()).isFalse(); + assertThat(rule.allTags()).containsOnly("bug", "performance", "cwe"); + assertThat(rule.createdAtAsLong()).isEqualTo(1500000000000L); + assertThat(rule.updatedAtAtAsLong()).isEqualTo(1600000000000L); + + rule = rulesByKey.get("S002"); + assertThat(rule.key()).isEqualTo(RuleKey.of("xoo", "S002")); + assertThat(rule.keyAsList()).containsOnly("xoo", "S002"); + assertThat(rule.ruleKey()).isEqualTo("S002"); + assertThat(rule.repository()).isEqualTo("xoo"); + assertThat(rule.internalKey()).isEqualTo("S2"); + assertThat(rule.name()).isEqualTo("Slow"); + assertThat(rule.htmlDescription()).isEqualTo("S002 desc"); + assertThat(rule.language()).isEqualTo("xoo"); + assertThat(rule.severity()).isEqualTo(Severity.CRITICAL); + assertThat(rule.status()).isEqualTo(RuleStatus.BETA); + assertThat(rule.isTemplate()).isTrue(); + assertThat(rule.allTags()).isEmpty(); + assertThat(rule.createdAtAsLong()).isEqualTo(2000000000000L); + assertThat(rule.updatedAtAtAsLong()).isEqualTo(2100000000000L); + } + + @Test + public void custom_rule() { + dbTester.prepareDbUnit(getClass(), "custom_rule.xml"); + + RuleResultSetIterator it = RuleResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), 0L); + Map rulesByKey = rulesByKey(it); + it.close(); + + assertThat(rulesByKey).hasSize(2); + + RuleDoc rule = rulesByKey.get("S001"); + assertThat(rule.isTemplate()).isTrue(); + assertThat(rule.templateKey()).isNull(); + + rule = rulesByKey.get("S002"); + assertThat(rule.isTemplate()).isFalse(); + assertThat(rule.templateKey()).isEqualTo(RuleKey.of("xoo", "S001")); + } + + @Test + public void removed_rule_is_returned() { + dbTester.prepareDbUnit(getClass(), "removed_rule.xml"); + RuleResultSetIterator it = RuleResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), 0L); + Map rulesByKey = rulesByKey(it); + it.close(); + + assertThat(rulesByKey).hasSize(1); + } + + private static Map rulesByKey(RuleResultSetIterator it) { + return Maps.uniqueIndex(it, new Function() { + @Override + public String apply(@Nonnull RuleDoc rule) { + return rule.key().rule(); + } + }); + } +} diff --git a/server/sonar-server/src/test/resources/org/sonar/server/rule/db/RuleDaoTest/insert-result.xml b/server/sonar-server/src/test/resources/org/sonar/server/rule/db/RuleDaoTest/insert-result.xml index 77e5d99adb8..4baf0a8327f 100644 --- a/server/sonar-server/src/test/resources/org/sonar/server/rule/db/RuleDaoTest/insert-result.xml +++ b/server/sonar-server/src/test/resources/org/sonar/server/rule/db/RuleDaoTest/insert-result.xml @@ -9,7 +9,7 @@ remediation_offset="5min" default_remediation_offset="10h" effort_to_fix_description="squid.S115.effortToFix" description_format="MARKDOWN" created_at="2013-12-16" updated_at="2013-12-16" - created_at_ms="[null]" updated_at_ms="[null]" + created_at_ms="0" updated_at_ms="0" /> diff --git a/server/sonar-server/src/test/resources/org/sonar/server/rule/db/RuleDaoTest/insert_all-result.xml b/server/sonar-server/src/test/resources/org/sonar/server/rule/db/RuleDaoTest/insert_all-result.xml index f063e486961..ce6e8da3e4a 100644 --- a/server/sonar-server/src/test/resources/org/sonar/server/rule/db/RuleDaoTest/insert_all-result.xml +++ b/server/sonar-server/src/test/resources/org/sonar/server/rule/db/RuleDaoTest/insert_all-result.xml @@ -8,7 +8,7 @@ remediation_coeff="1h" default_remediation_coeff="5d" remediation_offset="5min" default_remediation_offset="10h" effort_to_fix_description="squid.S115.effortToFix" description_format="HTML" - created_at_ms="[null]" updated_at_ms="[null]" + created_at_ms="0" updated_at_ms="0" /> diff --git a/server/sonar-server/src/test/resources/org/sonar/server/rule/db/RuleDaoTest/update-result.xml b/server/sonar-server/src/test/resources/org/sonar/server/rule/db/RuleDaoTest/update-result.xml index dec9f35bb1d..11fadab518f 100644 --- a/server/sonar-server/src/test/resources/org/sonar/server/rule/db/RuleDaoTest/update-result.xml +++ b/server/sonar-server/src/test/resources/org/sonar/server/rule/db/RuleDaoTest/update-result.xml @@ -10,7 +10,7 @@ remediation_coeff="1h" default_remediation_coeff="5d" remediation_offset="5min" default_remediation_offset="10h" effort_to_fix_description="squid.S115.effortToFix" description_format="MARKDOWN" - created_at_ms="1500000000000" updated_at_ms="1600000000000" + created_at_ms="1500000000000" updated_at_ms="0" /> + + + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/rule/index/RuleIndexerTest/removed_rule.xml b/server/sonar-server/src/test/resources/org/sonar/server/rule/index/RuleIndexerTest/removed_rule.xml new file mode 100644 index 00000000000..3ab588b95e2 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/rule/index/RuleIndexerTest/removed_rule.xml @@ -0,0 +1,13 @@ + + + + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/rule/index/RuleResultSetIteratorTest/custom_rule.xml b/server/sonar-server/src/test/resources/org/sonar/server/rule/index/RuleResultSetIteratorTest/custom_rule.xml new file mode 100644 index 00000000000..13f34bcd263 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/rule/index/RuleResultSetIteratorTest/custom_rule.xml @@ -0,0 +1,25 @@ + + + + + + + + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/rule/index/RuleResultSetIteratorTest/one_rule.xml b/server/sonar-server/src/test/resources/org/sonar/server/rule/index/RuleResultSetIteratorTest/one_rule.xml new file mode 100644 index 00000000000..b3e777b7735 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/rule/index/RuleResultSetIteratorTest/one_rule.xml @@ -0,0 +1,13 @@ + + + + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/rule/index/RuleResultSetIteratorTest/removed_rule.xml b/server/sonar-server/src/test/resources/org/sonar/server/rule/index/RuleResultSetIteratorTest/removed_rule.xml new file mode 100644 index 00000000000..3ab588b95e2 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/rule/index/RuleResultSetIteratorTest/removed_rule.xml @@ -0,0 +1,13 @@ + + + + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/rule/index/RuleResultSetIteratorTest/shared.xml b/server/sonar-server/src/test/resources/org/sonar/server/rule/index/RuleResultSetIteratorTest/shared.xml new file mode 100644 index 00000000000..65e7c7a0866 --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/rule/index/RuleResultSetIteratorTest/shared.xml @@ -0,0 +1,22 @@ + + + + + + diff --git a/sonar-db/src/test/resources/org/sonar/db/version/v55/FeedRulesLongDateColumnsTest/execute.xml b/sonar-db/src/test/resources/org/sonar/db/version/v55/FeedRulesLongDateColumnsTest/execute.xml index e46c1ac2ac2..bbc1f2416a5 100644 --- a/sonar-db/src/test/resources/org/sonar/db/version/v55/FeedRulesLongDateColumnsTest/execute.xml +++ b/sonar-db/src/test/resources/org/sonar/db/version/v55/FeedRulesLongDateColumnsTest/execute.xml @@ -8,7 +8,7 @@ created_at_ms="[null]" updated_at_ms="[null]" /> - +