123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629 |
- /*
- * SonarQube
- * Copyright (C) 2009-2022 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
- package org.sonar.server.rule.index;
-
- import com.google.common.base.Joiner;
- import java.util.ArrayList;
- import java.util.Arrays;
- import java.util.Collection;
- import java.util.HashMap;
- import java.util.Iterator;
- import java.util.List;
- import java.util.Map;
- import java.util.function.Function;
- import javax.annotation.Nullable;
- import org.apache.commons.lang.StringUtils;
- import org.apache.lucene.search.join.ScoreMode;
- import org.elasticsearch.action.search.SearchRequest;
- import org.elasticsearch.action.search.SearchResponse;
- import org.elasticsearch.core.TimeValue;
- import org.elasticsearch.index.query.BoolQueryBuilder;
- import org.elasticsearch.index.query.Operator;
- import org.elasticsearch.index.query.QueryBuilder;
- import org.elasticsearch.index.query.QueryBuilders;
- import org.elasticsearch.join.aggregations.JoinAggregationBuilders;
- import org.elasticsearch.join.query.HasParentQueryBuilder;
- import org.elasticsearch.join.query.JoinQueryBuilders;
- import org.elasticsearch.search.aggregations.AggregationBuilder;
- import org.elasticsearch.search.aggregations.AggregationBuilders;
- import org.elasticsearch.search.aggregations.BucketOrder;
- import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
- import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
- import org.elasticsearch.search.builder.SearchSourceBuilder;
- import org.elasticsearch.search.sort.FieldSortBuilder;
- import org.elasticsearch.search.sort.SortBuilders;
- import org.elasticsearch.search.sort.SortOrder;
- import org.sonar.api.rule.RuleStatus;
- import org.sonar.api.rule.Severity;
- import org.sonar.api.rules.RuleType;
- import org.sonar.api.utils.System2;
- import org.sonar.core.util.stream.MoreCollectors;
- import org.sonar.db.qualityprofile.QProfileDto;
- import org.sonar.server.es.EsClient;
- import org.sonar.server.es.EsUtils;
- import org.sonar.server.es.SearchIdResult;
- import org.sonar.server.es.SearchOptions;
- import org.sonar.server.es.StickyFacetBuilder;
- import org.sonar.server.es.newindex.DefaultIndexSettings;
- import org.sonar.server.es.textsearch.JavaTokenizer;
- import org.sonar.server.security.SecurityStandards;
-
- import static com.google.common.base.Preconditions.checkArgument;
- import static java.lang.Boolean.FALSE;
- import static java.lang.Boolean.TRUE;
- import static java.util.Collections.emptyList;
- import static java.util.Optional.ofNullable;
- import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
- import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
- import static org.elasticsearch.index.query.QueryBuilders.matchPhraseQuery;
- import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
- import static org.elasticsearch.index.query.QueryBuilders.termsQuery;
- import static org.sonar.api.rules.RuleType.SECURITY_HOTSPOT;
- import static org.sonar.api.rules.RuleType.VULNERABILITY;
- import static org.sonar.server.es.EsUtils.SCROLL_TIME_IN_MINUTES;
- import static org.sonar.server.es.EsUtils.optimizeScrollRequest;
- import static org.sonar.server.es.EsUtils.scrollIds;
- import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE;
- import static org.sonar.server.es.StickyFacetBuilder.FACET_DEFAULT_SIZE;
- import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.ENGLISH_HTML_ANALYZER;
- import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER;
- import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_WORDS_ANALYZER;
- import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
- import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_INHERITANCE;
- import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_PROFILE_UUID;
- import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_SEVERITY;
- import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_CREATED_AT;
- import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_CWE;
- import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_HTML_DESCRIPTION;
- import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_INTERNAL_KEY;
- import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_IS_EXTERNAL;
- import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_IS_TEMPLATE;
- import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_KEY;
- import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_LANGUAGE;
- import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_NAME;
- import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_OWASP_TOP_10;
- import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_OWASP_TOP_10_2021;
- import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_REPOSITORY;
- import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_RULE_KEY;
- import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_SANS_TOP_25;
- import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_SEVERITY;
- import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_SONARSOURCE_SECURITY;
- import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_STATUS;
- import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_TAGS;
- import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_TEMPLATE_KEY;
- import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_TYPE;
- import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_UPDATED_AT;
- import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_ACTIVE_RULE;
- import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE;
-
- /**
- * The unique entry-point to interact with Elasticsearch index "rules".
- * All the requests are listed here.
- */
- public class RuleIndex {
-
- public static final String FACET_LANGUAGES = "languages";
- public static final String FACET_TAGS = "tags";
- public static final String FACET_REPOSITORIES = "repositories";
- public static final String FACET_SEVERITIES = "severities";
- public static final String FACET_ACTIVE_SEVERITIES = "active_severities";
- public static final String FACET_STATUSES = "statuses";
- public static final String FACET_TYPES = "types";
- public static final String FACET_OLD_DEFAULT = "true";
- public static final String FACET_CWE = "cwe";
- public static final String FACET_SANS_TOP_25 = "sansTop25";
- public static final String FACET_OWASP_TOP_10 = "owaspTop10";
- public static final String FACET_OWASP_TOP_10_2021 = "owaspTop10-2021";
- public static final String FACET_SONARSOURCE_SECURITY = "sonarsourceSecurity";
-
- private static final int MAX_FACET_SIZE = 100;
-
- public static final List<String> ALL_STATUSES_EXCEPT_REMOVED = Arrays.stream(RuleStatus.values())
- .filter(status -> !RuleStatus.REMOVED.equals(status))
- .map(RuleStatus::toString)
- .collect(MoreCollectors.toList());
-
- private static final String AGGREGATION_NAME_FOR_TAGS = "tagsAggregation";
-
- private final EsClient client;
- private final System2 system2;
-
- public RuleIndex(EsClient client, System2 system2) {
- this.client = client;
- this.system2 = system2;
- }
-
- public SearchIdResult<String> search(RuleQuery query, SearchOptions options) {
- SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
-
- QueryBuilder qb = buildQuery(query);
- Map<String, QueryBuilder> filters = buildFilters(query);
-
- if (!options.getFacets().isEmpty()) {
- for (AggregationBuilder aggregation : getFacets(query, options, qb, filters).values()) {
- sourceBuilder.aggregation(aggregation);
- }
- }
-
- setSorting(query, sourceBuilder);
- setPagination(options, sourceBuilder);
-
- BoolQueryBuilder fb = boolQuery();
- for (QueryBuilder filterBuilder : filters.values()) {
- fb.must(filterBuilder);
- }
-
- sourceBuilder.query(boolQuery().must(qb).filter(fb));
-
- SearchRequest esSearch = EsClient.prepareSearch(TYPE_RULE)
- .source(sourceBuilder);
-
- return new SearchIdResult<>(client.search(esSearch), input -> input, system2.getDefaultTimeZone().toZoneId());
- }
-
- /**
- * Return all rule uuids matching the search query, without pagination nor facets
- */
- public Iterator<String> searchAll(RuleQuery query) {
- SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
-
- optimizeScrollRequest(sourceBuilder);
- QueryBuilder qb = buildQuery(query);
- Map<String, QueryBuilder> filters = buildFilters(query);
-
- BoolQueryBuilder fb = boolQuery();
- for (QueryBuilder filterBuilder : filters.values()) {
- fb.must(filterBuilder);
- }
-
- sourceBuilder.query(boolQuery().must(qb).filter(fb));
-
- SearchRequest esSearch = EsClient.prepareSearch(TYPE_RULE)
- .scroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES))
- .source(sourceBuilder);
-
- SearchResponse response = client.search(esSearch);
- return scrollIds(client, response, i -> i);
- }
-
- /* Build main query (search based) */
- private static QueryBuilder buildQuery(RuleQuery query) {
-
- // No contextual query case
- String queryText = query.getQueryText();
- if (StringUtils.isEmpty(queryText)) {
- return matchAllQuery();
- }
-
- // Build RuleBased contextual query
- BoolQueryBuilder qb = boolQuery();
-
- BoolQueryBuilder textQuery = boolQuery();
- JavaTokenizer.split(queryText)
- .stream().map(token -> boolQuery().should(
- matchQuery(
- SEARCH_GRAMS_ANALYZER.subField(FIELD_RULE_NAME),
- StringUtils.left(token, DefaultIndexSettings.MAXIMUM_NGRAM_LENGTH)).boost(20F))
- .should(
- matchPhraseQuery(
- ENGLISH_HTML_ANALYZER.subField(FIELD_RULE_HTML_DESCRIPTION),
- token).boost(3F)))
- .forEach(textQuery::must);
- qb.should(textQuery.boost(20F));
-
- // Match and partial Match queries
- // Search by key uses the "sortable" sub-field as it requires to be case-insensitive (lower-case filtering)
- qb.should(matchQuery(SORTABLE_ANALYZER.subField(FIELD_RULE_KEY), queryText).operator(Operator.AND).boost(30F));
- qb.should(matchQuery(SORTABLE_ANALYZER.subField(FIELD_RULE_RULE_KEY), queryText).operator(Operator.AND).boost(15F));
- qb.should(termQuery(FIELD_RULE_LANGUAGE, queryText, 3F));
- return qb;
- }
-
- private static QueryBuilder termQuery(String field, String query, float boost) {
- return QueryBuilders.multiMatchQuery(query,
- field, SEARCH_WORDS_ANALYZER.subField(field))
- .operator(Operator.AND)
- .boost(boost);
- }
-
- /* Build main filter (match based) */
- private static Map<String, QueryBuilder> buildFilters(RuleQuery query) {
- Map<String, QueryBuilder> filters = new HashMap<>();
-
- /* Add enforced filter on main type Rule */
- filters.put(
- FIELD_INDEX_TYPE,
- boolQuery().must(QueryBuilders.termsQuery(FIELD_INDEX_TYPE, TYPE_RULE.getType())));
-
- /* Add enforced filter on rules that are REMOVED */
- filters.put(FIELD_RULE_STATUS,
- boolQuery().mustNot(
- QueryBuilders.termQuery(FIELD_RULE_STATUS,
- RuleStatus.REMOVED.toString())));
-
- addFilter(filters, FIELD_RULE_INTERNAL_KEY, query.getInternalKey());
-
- addFilter(filters, FIELD_RULE_RULE_KEY, query.getRuleKey());
-
- addFilter(filters, query.getLanguages(), FIELD_RULE_LANGUAGE);
-
- addFilter(filters, query.getRepositories(), FIELD_RULE_REPOSITORY);
-
- addFilter(filters, query.getSeverities(), FIELD_RULE_SEVERITY);
-
- addSecurityStandardFilter(filters, FIELD_RULE_CWE, query.getCwe());
-
- addSecurityStandardFilter(filters, FIELD_RULE_OWASP_TOP_10, query.getOwaspTop10());
-
- addSecurityStandardFilter(filters, FIELD_RULE_OWASP_TOP_10_2021, query.getOwaspTop10For2021());
-
- addSecurityStandardFilter(filters, FIELD_RULE_SANS_TOP_25, query.getSansTop25());
-
- addSecurityStandardFilter(filters, FIELD_RULE_SONARSOURCE_SECURITY, query.getSonarsourceSecurity());
-
- addFilter(filters, FIELD_RULE_KEY, query.getKey());
-
- if (isNotEmpty(query.getTags())) {
- filters.put(FIELD_RULE_TAGS, buildTagsFilter(query.getTags()));
- }
-
- Collection<RuleType> types = query.getTypes();
- if (isNotEmpty(types)) {
- List<String> typeNames = types.stream().map(RuleType::toString).collect(MoreCollectors.toList());
- filters.put(FIELD_RULE_TYPE,
- QueryBuilders.termsQuery(FIELD_RULE_TYPE, typeNames));
- }
-
- if (query.getAvailableSinceLong() != null) {
- filters.put("availableSince", QueryBuilders.rangeQuery(FIELD_RULE_CREATED_AT)
- .gte(query.getAvailableSinceLong()));
- }
-
- if (isNotEmpty(query.getStatuses())) {
- Collection<String> stringStatus = new ArrayList<>();
- for (RuleStatus status : query.getStatuses()) {
- stringStatus.add(status.name());
- }
- filters.put(FIELD_RULE_STATUS,
- QueryBuilders.termsQuery(FIELD_RULE_STATUS, stringStatus));
- }
-
- Boolean isTemplate = query.isTemplate();
- if (isTemplate != null) {
- filters.put(FIELD_RULE_IS_TEMPLATE,
- QueryBuilders.termQuery(FIELD_RULE_IS_TEMPLATE, Boolean.toString(isTemplate)));
- }
-
- boolean includeExternal = query.includeExternal();
- if (!includeExternal) {
- filters.put(FIELD_RULE_IS_EXTERNAL,
- QueryBuilders.termQuery(FIELD_RULE_IS_EXTERNAL, false));
- }
-
- String template = query.templateKey();
- if (template != null) {
- filters.put(FIELD_RULE_TEMPLATE_KEY,
- QueryBuilders.termQuery(FIELD_RULE_TEMPLATE_KEY, template));
- }
-
- /* Implementation of activation query */
- QProfileDto profile = query.getQProfile();
- if (query.getActivation() != null && profile != null) {
- QueryBuilder childQuery = buildActivationFilter(query, profile);
-
- if (TRUE.equals(query.getActivation())) {
- filters.put("activation",
- JoinQueryBuilders.hasChildQuery(TYPE_ACTIVE_RULE.getName(),
- childQuery, ScoreMode.None));
- } else if (FALSE.equals(query.getActivation())) {
- filters.put("activation",
- boolQuery().mustNot(
- JoinQueryBuilders.hasChildQuery(TYPE_ACTIVE_RULE.getName(),
- childQuery, ScoreMode.None)));
- }
- QProfileDto compareToQProfile = query.getCompareToQProfile();
- if (compareToQProfile != null) {
- filters.put("comparison",
- JoinQueryBuilders.hasChildQuery(
- TYPE_ACTIVE_RULE.getName(),
- boolQuery().must(QueryBuilders.termQuery(FIELD_ACTIVE_RULE_PROFILE_UUID, compareToQProfile.getRulesProfileUuid())),
- ScoreMode.None));
- }
- }
-
- return filters;
- }
-
- private static void addSecurityStandardFilter(Map<String, QueryBuilder> filters, String key, Collection<String> values) {
- if (isNotEmpty(values)) {
- filters.put(key,
- boolQuery()
- .must(QueryBuilders.termsQuery(key, values))
- .must(QueryBuilders.termsQuery(FIELD_RULE_TYPE, VULNERABILITY.name(), SECURITY_HOTSPOT.name())));
- }
- }
-
- private static void addFilter(Map<String, QueryBuilder> filters, Collection<String> key, String value) {
- if (isNotEmpty(key)) {
- filters.put(value,
- QueryBuilders.termsQuery(value, key));
- }
- }
-
- private static void addFilter(Map<String, QueryBuilder> filters, String key, String value) {
- if (StringUtils.isNotEmpty(value)) {
- filters.put(key,
- QueryBuilders.termQuery(key, value));
- }
- }
-
- private static BoolQueryBuilder buildTagsFilter(Collection<String> tags) {
- BoolQueryBuilder q = boolQuery();
- for (String tag : tags) {
- q.should(boolQuery().filter(QueryBuilders.termQuery(FIELD_RULE_TAGS, tag)));
- }
- return q;
- }
-
- private static QueryBuilder buildActivationFilter(RuleQuery query, QProfileDto profile) {
- // ActiveRule Filter (profile and inheritance)
- BoolQueryBuilder activeRuleFilter = boolQuery();
- addTermFilter(activeRuleFilter, FIELD_ACTIVE_RULE_PROFILE_UUID, profile.getRulesProfileUuid());
- addTermFilter(activeRuleFilter, FIELD_ACTIVE_RULE_INHERITANCE, query.getInheritance());
- addTermFilter(activeRuleFilter, FIELD_ACTIVE_RULE_SEVERITY, query.getActiveSeverities());
-
- // ChildQuery
- QueryBuilder childQuery;
- if (activeRuleFilter.hasClauses()) {
- childQuery = activeRuleFilter;
- } else {
- childQuery = matchAllQuery();
- }
- return childQuery;
- }
-
- private static BoolQueryBuilder addTermFilter(BoolQueryBuilder filter, String field, @Nullable Collection<String> values) {
- if (isNotEmpty(values)) {
- BoolQueryBuilder valuesFilter = boolQuery();
- for (String value : values) {
- QueryBuilder valueFilter = QueryBuilders.termQuery(field, value);
- valuesFilter.should(valueFilter);
- }
- filter.must(valuesFilter);
- }
- return filter;
- }
-
- private static BoolQueryBuilder addTermFilter(BoolQueryBuilder filter, String field, @Nullable String value) {
- if (StringUtils.isNotEmpty(value)) {
- filter.must(QueryBuilders.termQuery(field, value));
- }
- return filter;
- }
-
- private static Map<String, AggregationBuilder> getFacets(RuleQuery query, SearchOptions options, QueryBuilder queryBuilder, Map<String, QueryBuilder> filters) {
- Map<String, AggregationBuilder> aggregations = new HashMap<>();
- StickyFacetBuilder stickyFacetBuilder = stickyFacetBuilder(queryBuilder, filters);
-
- addDefaultFacets(query, options, aggregations, stickyFacetBuilder);
-
- addStatusFacetIfNeeded(options, aggregations, stickyFacetBuilder);
-
- if (options.getFacets().contains(FACET_SEVERITIES)) {
- aggregations.put(FACET_SEVERITIES,
- stickyFacetBuilder.buildStickyFacet(FIELD_RULE_SEVERITY, FACET_SEVERITIES, Severity.ALL.toArray()));
- }
-
- addActiveSeverityFacetIfNeeded(query, options, aggregations, stickyFacetBuilder);
- return aggregations;
- }
-
- private static void addDefaultFacets(RuleQuery query, SearchOptions options, Map<String, AggregationBuilder> aggregations, StickyFacetBuilder stickyFacetBuilder) {
- if (options.getFacets().contains(FACET_LANGUAGES) || options.getFacets().contains(FACET_OLD_DEFAULT)) {
- Collection<String> languages = query.getLanguages();
- aggregations.put(FACET_LANGUAGES,
- stickyFacetBuilder.buildStickyFacet(FIELD_RULE_LANGUAGE, FACET_LANGUAGES, MAX_FACET_SIZE,
- (languages == null) ? (new String[0]) : languages.toArray()));
- }
- if (options.getFacets().contains(FACET_TAGS) || options.getFacets().contains(FACET_OLD_DEFAULT)) {
- Collection<String> tags = query.getTags();
- aggregations.put(FACET_TAGS,
- stickyFacetBuilder.buildStickyFacet(FIELD_RULE_TAGS, FACET_TAGS, MAX_FACET_SIZE,
- (tags == null) ? (new String[0]) : tags.toArray()));
- }
- if (options.getFacets().contains(FACET_TYPES)) {
- Collection<RuleType> types = query.getTypes();
- aggregations.put(FACET_TYPES,
- stickyFacetBuilder.buildStickyFacet(FIELD_RULE_TYPE, FACET_TYPES,
- (types == null) ? (new String[0]) : types.toArray()));
- }
- if (options.getFacets().contains(FACET_REPOSITORIES) || options.getFacets().contains(FACET_OLD_DEFAULT)) {
- Collection<String> repositories = query.getRepositories();
- aggregations.put(FACET_REPOSITORIES,
- stickyFacetBuilder.buildStickyFacet(FIELD_RULE_REPOSITORY, FACET_REPOSITORIES, MAX_FACET_SIZE,
- (repositories == null) ? (new String[0]) : repositories.toArray()));
- }
-
- addDefaultSecurityFacets(query, options, aggregations, stickyFacetBuilder);
- }
-
- private static Function<TermsAggregationBuilder, AggregationBuilder> filterSecurityCategories() {
- return termsAggregation -> AggregationBuilders.filter(
- "filter_by_rule_types_" + termsAggregation.getName(),
- termsQuery(FIELD_RULE_TYPE,
- VULNERABILITY.name(),
- SECURITY_HOTSPOT.name()))
- .subAggregation(termsAggregation);
- }
-
- private static void addDefaultSecurityFacets(RuleQuery query, SearchOptions options, Map<String, AggregationBuilder> aggregations,
- StickyFacetBuilder stickyFacetBuilder) {
- if (options.getFacets().contains(FACET_CWE)) {
- Collection<String> categories = query.getCwe();
- aggregations.put(FACET_CWE,
- stickyFacetBuilder.buildStickyFacet(FIELD_RULE_CWE, FACET_CWE,
- FACET_DEFAULT_SIZE, filterSecurityCategories(),
- (categories == null) ? (new String[0]) : categories.toArray()));
- }
- if (options.getFacets().contains(FACET_OWASP_TOP_10)) {
- Collection<String> categories = query.getOwaspTop10();
- aggregations.put(FACET_OWASP_TOP_10,
- stickyFacetBuilder.buildStickyFacet(FIELD_RULE_OWASP_TOP_10, FACET_OWASP_TOP_10,
- FACET_DEFAULT_SIZE, filterSecurityCategories(),
- (categories == null) ? (new String[0]) : categories.toArray()));
- }
- if (options.getFacets().contains(FACET_OWASP_TOP_10_2021)) {
- Collection<String> categories = query.getOwaspTop10For2021();
- aggregations.put(FACET_OWASP_TOP_10_2021,
- stickyFacetBuilder.buildStickyFacet(FIELD_RULE_OWASP_TOP_10_2021, FACET_OWASP_TOP_10_2021,
- FACET_DEFAULT_SIZE, filterSecurityCategories(),
- (categories == null) ? (new String[0]) : categories.toArray()));
- }
- if (options.getFacets().contains(FACET_SANS_TOP_25)) {
- Collection<String> categories = query.getSansTop25();
- aggregations.put(FACET_SANS_TOP_25,
- stickyFacetBuilder.buildStickyFacet(FIELD_RULE_SANS_TOP_25, FACET_SANS_TOP_25,
- FACET_DEFAULT_SIZE, filterSecurityCategories(),
- (categories == null) ? (new String[0]) : categories.toArray()));
- }
- if (options.getFacets().contains(FACET_SONARSOURCE_SECURITY)) {
- Collection<String> categories = query.getSonarsourceSecurity();
- aggregations.put(FACET_SONARSOURCE_SECURITY,
- stickyFacetBuilder.buildStickyFacet(FIELD_RULE_SONARSOURCE_SECURITY, FACET_SONARSOURCE_SECURITY,
- SecurityStandards.SQCategory.values().length, filterSecurityCategories(),
- (categories == null) ? (new String[0]) : categories.toArray()));
- }
- }
-
- private static void addStatusFacetIfNeeded(SearchOptions options, Map<String, AggregationBuilder> aggregations, StickyFacetBuilder stickyFacetBuilder) {
- if (options.getFacets().contains(FACET_STATUSES)) {
- BoolQueryBuilder facetFilter = stickyFacetBuilder.getStickyFacetFilter(FIELD_RULE_STATUS);
- AggregationBuilder statuses = AggregationBuilders.filter(FACET_STATUSES + "_filter", facetFilter)
- .subAggregation(
- AggregationBuilders
- .terms(FACET_STATUSES)
- .field(FIELD_RULE_STATUS)
- .includeExclude(new IncludeExclude(Joiner.on('|').join(ALL_STATUSES_EXCEPT_REMOVED), RuleStatus.REMOVED.toString()))
- .size(ALL_STATUSES_EXCEPT_REMOVED.size()));
-
- aggregations.put(FACET_STATUSES, AggregationBuilders.global(FACET_STATUSES).subAggregation(statuses));
- }
- }
-
- private static void addActiveSeverityFacetIfNeeded(RuleQuery query, SearchOptions options, Map<String, AggregationBuilder> aggregations,
- StickyFacetBuilder stickyFacetBuilder) {
- QProfileDto profile = query.getQProfile();
- if (options.getFacets().contains(FACET_ACTIVE_SEVERITIES) && profile != null) {
- // We are building a children aggregation on active rules
- // so the rule filter has to be used as parent filter for active rules
- // from which we remove filters that concern active rules ("activation")
- HasParentQueryBuilder ruleFilter = JoinQueryBuilders.hasParentQuery(
- TYPE_RULE.getType(),
- stickyFacetBuilder.getStickyFacetFilter("activation"),
- false);
-
- // Rebuilding the active rule filter without severities
- BoolQueryBuilder childrenFilter = boolQuery();
- addTermFilter(childrenFilter, FIELD_ACTIVE_RULE_PROFILE_UUID, profile.getRulesProfileUuid());
- RuleIndex.addTermFilter(childrenFilter, FIELD_ACTIVE_RULE_INHERITANCE, query.getInheritance());
- QueryBuilder activeRuleFilter = childrenFilter.must(ruleFilter);
-
- AggregationBuilder activeSeverities = JoinAggregationBuilders.children(FACET_ACTIVE_SEVERITIES + "_children", TYPE_ACTIVE_RULE.getName())
- .subAggregation(
- AggregationBuilders.filter(FACET_ACTIVE_SEVERITIES + "_filter", activeRuleFilter)
- .subAggregation(
- AggregationBuilders
- .terms(FACET_ACTIVE_SEVERITIES)
- .field(FIELD_ACTIVE_RULE_SEVERITY)
- .includeExclude(new IncludeExclude(Joiner.on('|').join(Severity.ALL), null))
- .size(Severity.ALL.size())));
-
- aggregations.put(FACET_ACTIVE_SEVERITIES, AggregationBuilders.global(FACET_ACTIVE_SEVERITIES).subAggregation(activeSeverities));
- }
- }
-
- private static StickyFacetBuilder stickyFacetBuilder(QueryBuilder query, Map<String, QueryBuilder> filters) {
- return new StickyFacetBuilder(query, filters, null, BucketOrder.compound(BucketOrder.count(false), BucketOrder.key(true)));
- }
-
- private static void setSorting(RuleQuery query, SearchSourceBuilder esSearch) {
- /* integrate Query Sort */
- String queryText = query.getQueryText();
- if (query.getSortField() != null) {
- FieldSortBuilder sort = SortBuilders.fieldSort(appendSortSuffixIfNeeded(query.getSortField()));
- if (query.isAscendingSort()) {
- sort.order(SortOrder.ASC);
- } else {
- sort.order(SortOrder.DESC);
- }
- esSearch.sort(sort);
- } else if (StringUtils.isNotEmpty(queryText)) {
- esSearch.sort(SortBuilders.scoreSort());
- } else {
- esSearch.sort(appendSortSuffixIfNeeded(FIELD_RULE_UPDATED_AT), SortOrder.DESC);
- // deterministic sort when exactly the same updated_at (same millisecond)
- esSearch.sort(appendSortSuffixIfNeeded(FIELD_RULE_KEY), SortOrder.ASC);
- }
- }
-
- private static String appendSortSuffixIfNeeded(String field) {
- return field +
- ((field.equals(FIELD_RULE_NAME) || field.equals(FIELD_RULE_KEY))
- ? ("." + SORTABLE_ANALYZER.getSubFieldSuffix())
- : "");
- }
-
- private static void setPagination(SearchOptions options, SearchSourceBuilder esSearch) {
- esSearch.from(options.getOffset());
- esSearch.size(options.getLimit());
- }
-
- public List<String> listTags(@Nullable String query, int size) {
- int maxPageSize = 500;
- checkArgument(size <= maxPageSize, "Page size must be lower than or equals to " + maxPageSize);
- if (size <= 0) {
- return emptyList();
- }
-
- TermsAggregationBuilder termsAggregation = AggregationBuilders.terms(AGGREGATION_NAME_FOR_TAGS)
- .field(FIELD_RULE_TAGS)
- .size(size)
- .order(BucketOrder.key(true))
- .minDocCount(1);
- ofNullable(query)
- .map(EsUtils::escapeSpecialRegexChars)
- .map(queryString -> ".*" + queryString + ".*")
- .map(s -> new IncludeExclude(s, null))
- .ifPresent(termsAggregation::includeExclude);
-
- SearchRequest request = EsClient.prepareSearch(TYPE_RULE.getMainType())
- .source(new SearchSourceBuilder()
- .query(matchAllQuery())
- .size(0)
- .aggregation(termsAggregation));
-
- SearchResponse esResponse = client.search(request);
- return EsUtils.termsKeys(esResponse.getAggregations().get(AGGREGATION_NAME_FOR_TAGS));
- }
-
- private static boolean isNotEmpty(@Nullable Collection<?> list) {
- return list != null && !list.isEmpty();
- }
- }
|