From 2a1f7ca4e6db457852042e96d1379d8ebd547792 Mon Sep 17 00:00:00 2001 From: Jacek Date: Thu, 19 Mar 2020 13:08:00 +0100 Subject: [PATCH] SONAR-13104 drop global aggregation from issues ES searches --- .../main/java/org/sonar/server/es/Facets.java | 3 +- .../sonar/server/es/StickyFacetBuilder.java | 4 - .../searchrequest/RequestFiltersComputer.java | 266 ++++++++ .../searchrequest/SubAggregationHelper.java | 97 +++ .../searchrequest/TermTopAggregationDef.java | 55 ++ .../es/searchrequest/TopAggregationDef.java | 48 ++ .../TopAggregationDefinition.java | 30 + .../searchrequest/TopAggregationHelper.java | 95 +++ .../server/es/searchrequest/package-info.java | 23 + .../es/searchrequest/AllFiltersTest.java | 119 ++++ .../RequestFiltersComputerTest.java | 276 ++++++++ .../SubAggregationHelperTest.java | 164 +++++ .../TermTopAggregationDefTest.java | 89 +++ .../searchrequest/TopAggregationDefTest.java | 52 ++ .../TopAggregationHelperTest.java | 236 +++++++ .../sonar/server/issue/index/IssueIndex.java | 604 ++++++++++-------- 16 files changed, 1888 insertions(+), 273 deletions(-) create mode 100644 server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/RequestFiltersComputer.java create mode 100644 server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/SubAggregationHelper.java create mode 100644 server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/TermTopAggregationDef.java create mode 100644 server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/TopAggregationDef.java create mode 100644 server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/TopAggregationDefinition.java create mode 100644 server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/TopAggregationHelper.java create mode 100644 server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/package-info.java create mode 100644 server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/AllFiltersTest.java create mode 100644 server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/RequestFiltersComputerTest.java create mode 100644 server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/SubAggregationHelperTest.java create mode 100644 server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/TermTopAggregationDefTest.java create mode 100644 server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/TopAggregationDefTest.java create mode 100644 server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/TopAggregationHelperTest.java diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/Facets.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/Facets.java index 110c4601b04..3aa00c816da 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/Facets.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/Facets.java @@ -45,6 +45,7 @@ import static org.sonarqube.ws.client.issue.IssuesWsParameters.FACET_MODE_EFFORT public class Facets { + public static final String SELECTED_SUB_AGG_NAME_SUFFIX = "_selected"; public static final String TOTAL = "total"; private static final java.lang.String NO_DATA_PREFIX = "no_data_"; @@ -105,7 +106,7 @@ public class Facets { if (facetName.contains("__") && !facetName.startsWith("__")) { facetName = facetName.substring(0, facetName.indexOf("__")); } - facetName = facetName.replace("_selected", ""); + facetName = facetName.replace(SELECTED_SUB_AGG_NAME_SUFFIX, ""); LinkedHashMap facet = getOrCreateFacet(facetName); for (Terms.Bucket value : aggregation.getBuckets()) { List aggregationList = value.getAggregations().asList(); diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/StickyFacetBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/StickyFacetBuilder.java index ef3af1381a4..9ba5e34a0f5 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/StickyFacetBuilder.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/StickyFacetBuilder.java @@ -110,10 +110,6 @@ public class StickyFacetBuilder { return facetFilter; } - public FilterAggregationBuilder buildTopFacetAggregation(String fieldName, String facetName, BoolQueryBuilder facetFilter, int size) { - return buildTopFacetAggregation(fieldName, facetName, facetFilter, size, t -> t); - } - private FilterAggregationBuilder buildTopFacetAggregation(String fieldName, String facetName, BoolQueryBuilder facetFilter, int size, Function additionalAggregationFilter) { TermsAggregationBuilder termsAggregation = buildTermsFacetAggregation(fieldName, facetName, size); diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/RequestFiltersComputer.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/RequestFiltersComputer.java new file mode 100644 index 00000000000..9c2d6c5c02a --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/RequestFiltersComputer.java @@ -0,0 +1,266 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.searchrequest; + +import com.google.common.collect.ImmutableSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.BiPredicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.sonar.core.util.stream.MoreCollectors; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; +import static java.util.Optional.empty; +import static java.util.Optional.of; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; + +/** + * Computes filters of a given ES search request given all the filters to apply and the top-aggregations to include in + * the request: + *
    + *
  • the ones for the query (see {@link #computeQueryFilter(AllFiltersImpl, Map) computeQueryFilter})
    • + *
  • the ones to apply as post filters (see {@link #computePostFilters(AllFiltersImpl, Set) computePostFilters})
    • + *
  • the ones for each top-aggregation (see {@link #getTopAggregationFilter(TopAggregationDefinition) getTopAggregationFilter})
    • + *
+ *

+ * To be able to provide accurate filters, all {@link TopAggregationDefinition} instances for which + * {@link #getTopAggregationFilter(TopAggregationDefinition)} may be called, must be declared in the constructor. + */ +public class RequestFiltersComputer { + + private final Set topAggregations; + private final Map postFilters; + private final Map queryFilters; + + public RequestFiltersComputer(AllFilters allFilters, Set topAggregations) { + this.topAggregations = ImmutableSet.copyOf(topAggregations); + this.postFilters = computePostFilters((AllFiltersImpl) allFilters, topAggregations); + this.queryFilters = computeQueryFilter((AllFiltersImpl) allFilters, postFilters); + } + + public static AllFilters newAllFilters() { + return new AllFiltersImpl(); + } + + /** + * Any filter of the query which can not be applied to all top-aggregations must be applied as a PostFilter. + *

+ * A filter applying to some field can not be applied to the query when at least one sticky top-aggregation is enabled + * which applies to that field and a top-aggregation is enabled on that field. + */ + private static Map computePostFilters(AllFiltersImpl allFilters, + Set topAggregations) { + Set enabledStickyTopAggregationtedFieldNames = topAggregations.stream() + .filter(TopAggregationDefinition::isSticky) + .map(TopAggregationDefinition::getFieldName) + .collect(MoreCollectors.toSet(topAggregations.size())); + + // use LinkedHashMap over MoreCollectors.uniqueIndex to preserve order and write UTs more easily + Map res = new LinkedHashMap<>(); + allFilters.internalStream() + .filter(e -> enabledStickyTopAggregationtedFieldNames.contains(e.getKey().getFieldName())) + .forEach(e -> checkState(res.put(e.getKey(), e.getValue()) == null, "Duplicate: %s", e.getKey())); + return res; + } + + /** + * Filters which can be applied directly to the query are only the filters which can also be applied to all + * aggregations. + *

+ * Aggregations are scoped by the filter of the query. If any top-aggregation need to not be applied a filter + * (typical case is a filter on the field aggregated to implement sticky facet behavior), this filter can + * not be applied to the query and therefor must be applied as PostFilter. + */ + private static Map computeQueryFilter(AllFiltersImpl allFilters, + Map postFilters) { + Set postFilterKeys = postFilters.keySet(); + + // use LinkedHashMap over MoreCollectors.uniqueIndex to preserve order and write UTs more easily + Map res = new LinkedHashMap<>(); + allFilters.internalStream() + .filter(e -> !postFilterKeys.contains(e.getKey())) + .forEach(e -> checkState(res.put(e.getKey(), e.getValue()) == null, "Duplicate: %s", e.getKey())); + return res; + } + + /** + * The {@link BoolQueryBuilder} to apply directly to the query in the ES request. + *

+ * There could be no filter to apply to the query in the (unexpected but supported) case where all filters + * need to be applied as PostFilter because none of them can be applied to all top-aggregations. + */ + public Optional getQueryFilters() { + return toBoolQuery(this.queryFilters, (e, v) -> true); + } + + /** + * The {@link BoolQueryBuilder} to add to the ES request as PostFilter + * (see {@link org.elasticsearch.action.search.SearchRequestBuilder#setPostFilter(QueryBuilder)}). + *

+ * There may be no PostFilter to apply at all. Typical case is when all filters apply to both the query and + * all aggregations. (corner case: when there is no filter at all...) + */ + public Optional getPostFilters() { + return toBoolQuery(postFilters, (e, v) -> true); + } + + /** + * The {@link BoolQueryBuilder} to apply to the top aggregation for the specified {@link TopAggregationDef}. + *

+ * The filter of the aggregations for a top-aggregation will either be: + *

    + *
  • the same as PostFilter, if the top-aggregation is non-sticky or the field the top-aggregation applies + * to is not being filtered
    • + *
  • or the same as PostFilter minus any filter which applies to the field for the top-aggregation (if it's sticky)
    • + *
+ * + * @throws IllegalArgumentException if specified {@link TopAggregationDefinition} has not been specified in the constructor + */ + public Optional getTopAggregationFilter(TopAggregationDefinition topAggregation) { + checkArgument(topAggregations.contains(topAggregation), "topAggregation must have been declared in constructor"); + return toBoolQuery( + postFilters, + (e, v) -> !topAggregation.isSticky() || !topAggregation.getFieldName().equals(e.getFieldName())); + } + + private static Optional toBoolQuery(Map queryFilters, + BiPredicate predicate) { + if (queryFilters.isEmpty()) { + return empty(); + } + + List selectQueryBuilders = queryFilters.entrySet().stream() + .filter(e -> predicate.test(e.getKey(), e.getValue())) + .map(Map.Entry::getValue) + .collect(Collectors.toList()); + if (selectQueryBuilders.isEmpty()) { + return empty(); + } + + BoolQueryBuilder res = boolQuery(); + selectQueryBuilders.forEach(res::must); + return of(res); + } + + /** + * A mean to put together all filters which apply to a given Search request. + */ + public interface AllFilters { + + /** + * @throws IllegalArgumentException if a filter with the specified name has already been added + */ + AllFilters addFilter(String name, String fieldName, @Nullable QueryBuilder filter); + + /** + * Convenience method for usage of {@link #addFilter(String, String, QueryBuilder)} when name of the filter is + * the same as the field name. + */ + AllFilters addFilter(String fieldName, @Nullable QueryBuilder filter); + + Stream stream(); + } + + private static class AllFiltersImpl implements AllFilters { + /** + * Usage of LinkedHashMap only benefits unit tests by providing predictability of the order of the filters. + * ES doesn't care of the order. + */ + private final Map filters = new LinkedHashMap<>(); + + @Override + public AllFilters addFilter(String fieldName, @Nullable QueryBuilder filter) { + return addFilter(fieldName, fieldName, filter); + } + + @Override + public AllFilters addFilter(String name, String fieldName, @Nullable QueryBuilder filter) { + requireNonNull(name, "name can't be null"); + requireNonNull(fieldName, "fieldName can't be null"); + + if (filter == null) { + return this; + } + + checkArgument( + filters.put(new FilterNameAndFieldName(name, fieldName), filter) == null, + "A filter with name %s has already been added", name); + return this; + } + + @Override + public Stream stream() { + return filters.values().stream(); + } + + private Stream> internalStream() { + return filters.entrySet().stream(); + } + } + + /** + * Serves as a key in internal map of filters, it behaves the same as if the filterName was directly used as a key in + * this map but also holds the name of the field each filter applies to. + *

+ * This saves from using two internal maps. + */ + @Immutable + private static final class FilterNameAndFieldName { + private final String filterName; + private final String fieldName; + + public FilterNameAndFieldName(String filterName, String fieldName) { + this.filterName = filterName; + this.fieldName = fieldName; + } + + public String getFieldName() { + return fieldName; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + FilterNameAndFieldName that = (FilterNameAndFieldName) o; + return filterName.equals(that.filterName); + } + + @Override + public int hashCode() { + return Objects.hash(filterName); + } + } +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/SubAggregationHelper.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/SubAggregationHelper.java new file mode 100644 index 00000000000..fb83c136caf --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/SubAggregationHelper.java @@ -0,0 +1,97 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.searchrequest; + +import java.util.Arrays; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collector; +import java.util.stream.Collectors; +import javax.annotation.CheckForNull; +import javax.annotation.Nullable; +import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.BucketOrder; +import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.sonar.server.es.EsUtils; +import org.sonar.server.es.Facets; + +import static java.lang.Math.max; +import static java.util.Optional.of; + +public class SubAggregationHelper { + private static final int TERM_AGGREGATION_MIN_DOC_COUNT = 1; + private static final BucketOrder ORDER_BY_BUCKET_SIZE_DESC = BucketOrder.count(false); + /** In some cases the user selects >15 items for one facet. In that case, we want to calculate the doc count for all of them (not just the first 15 items, which would be the + * default for the TermsAggregation). */ + private static final int MAXIMUM_NUMBER_OF_SELECTED_ITEMS_WHOSE_DOC_COUNT_WILL_BE_CALCULATED = 50; + private static final Collector PIPE_JOINER = Collectors.joining("|"); + + @CheckForNull + private final AbstractAggregationBuilder subAggregation; + private final BucketOrder order; + + public SubAggregationHelper() { + this(null, null); + } + + public SubAggregationHelper(@Nullable AbstractAggregationBuilder subAggregation) { + this(subAggregation, null); + } + + public SubAggregationHelper(@Nullable AbstractAggregationBuilder subAggregation, @Nullable BucketOrder order) { + this.subAggregation = subAggregation; + this.order = order == null ? ORDER_BY_BUCKET_SIZE_DESC : order; + } + + public TermsAggregationBuilder buildTermsAggregation(String name, TermTopAggregationDef topAggregation) { + TermsAggregationBuilder termsAggregation = AggregationBuilders.terms(name) + .field(topAggregation.getFieldName()) + .order(order) + .minDocCount(TERM_AGGREGATION_MIN_DOC_COUNT); + topAggregation.getMaxTerms().ifPresent(termsAggregation::size); + if (subAggregation != null) { + termsAggregation = termsAggregation.subAggregation(subAggregation); + } + return termsAggregation; + } + + public Optional buildSelectedItemsAggregation(String name, TopAggregationDefinition topAggregation, T[] selected) { + if (selected.length <= 0) { + return Optional.empty(); + } + + String includes = Arrays.stream(selected) + .filter(Objects::nonNull) + .map(s -> EsUtils.escapeSpecialRegexChars(s.toString())) + .collect(PIPE_JOINER); + + TermsAggregationBuilder selectedTerms = AggregationBuilders.terms(name + Facets.SELECTED_SUB_AGG_NAME_SUFFIX) + .size(max(MAXIMUM_NUMBER_OF_SELECTED_ITEMS_WHOSE_DOC_COUNT_WILL_BE_CALCULATED, includes.length())) + .field(topAggregation.getFieldName()) + .includeExclude(new IncludeExclude(includes, null)); + if (subAggregation != null) { + selectedTerms = selectedTerms.subAggregation(subAggregation); + } + + return of(selectedTerms); + } +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/TermTopAggregationDef.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/TermTopAggregationDef.java new file mode 100644 index 00000000000..10fc0d4401d --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/TermTopAggregationDef.java @@ -0,0 +1,55 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.searchrequest; + +import java.util.OptionalInt; +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +import static com.google.common.base.Preconditions.checkArgument; + +/** + * A top-aggregation which defines (at least) one sub-aggregation on the terms of the fields of the top-aggregation. + */ +@Immutable +public class TermTopAggregationDef implements TopAggregationDefinition { + private final TopAggregationDef delegate; + private final Integer maxTerms; + + public TermTopAggregationDef(String fieldName, boolean sticky, @Nullable Integer maxTerms) { + this.delegate = new TopAggregationDef(fieldName, sticky); + checkArgument(maxTerms == null || maxTerms >= 0, "maxTerms can't be < 0"); + this.maxTerms = maxTerms; + } + + @Override + public String getFieldName() { + return delegate.getFieldName(); + } + + @Override + public boolean isSticky() { + return delegate.isSticky(); + } + + public OptionalInt getMaxTerms() { + return maxTerms == null ? OptionalInt.empty() : OptionalInt.of(maxTerms); + } +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/TopAggregationDef.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/TopAggregationDef.java new file mode 100644 index 00000000000..57f61b837be --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/TopAggregationDef.java @@ -0,0 +1,48 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.searchrequest; + +import javax.annotation.concurrent.Immutable; + +import static java.util.Objects.requireNonNull; + +/** + * Default implementation of {@link TopAggregationDefinition}. + */ +@Immutable +public final class TopAggregationDef implements TopAggregationDefinition { + private final String fieldName; + private final boolean sticky; + + public TopAggregationDef(String fieldName, boolean sticky) { + this.fieldName = requireNonNull(fieldName, "fieldName can't be null"); + this.sticky = sticky; + } + + @Override + public String getFieldName() { + return fieldName; + } + + @Override + public boolean isSticky() { + return sticky; + } +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/TopAggregationDefinition.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/TopAggregationDefinition.java new file mode 100644 index 00000000000..609ade262ed --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/TopAggregationDefinition.java @@ -0,0 +1,30 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.searchrequest; + +/** + * Models a first level aggregation in an Elasticsearch request (aka. top-aggregation) on a unique field and whether + * it is to be used to compute data for a sticky facet (see {@link #isSticky()}). + */ +public interface TopAggregationDefinition { + String getFieldName(); + + boolean isSticky(); +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/TopAggregationHelper.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/TopAggregationHelper.java new file mode 100644 index 00000000000..035ea059977 --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/TopAggregationHelper.java @@ -0,0 +1,95 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.searchrequest; + +import java.util.function.Consumer; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; + +import static com.google.common.base.Preconditions.checkState; + +public class TopAggregationHelper { + + public static final Consumer NO_EXTRA_FILTER = t -> { + }; + public static final Consumer NO_OTHER_SUBAGGREGATION = t -> { + }; + + private final RequestFiltersComputer filterComputer; + private final SubAggregationHelper subAggregationHelper; + + public TopAggregationHelper(RequestFiltersComputer filterComputer, SubAggregationHelper subAggregationHelper) { + this.filterComputer = filterComputer; + this.subAggregationHelper = subAggregationHelper; + } + + /** + * Creates a top-level aggregation that will be correctly scoped (ie. filtered) to aggregate on + * {@code TopAggregationDefinition#getFieldName} given the Request filters and the other top-aggregations + * (see {@link RequestFiltersComputer#getTopAggregationFilter(TopAggregationDefinition)}). + *

+ * Optionally, the scope (ie. filter) of the aggregation can be further reduced by providing {@code extraFilters}. + *

+ * Aggregations must be added to the top-level one by providing {@code subAggregations} otherwise + * the aggregation will be empty and will yield no result. + * + * @param topAggregationName the name of the top-aggregation in the request + * @param topAggregation properties of the top-aggregation + * @param extraFilters optional extra filters which could further restrict the scope of computation of the + * top-terms aggregation + * @param subAggregations sub aggregation(s) to actually compute something + * + * @throws IllegalStateException if no sub-aggregation has been added + * @return the aggregation, that can be added on top level of the elasticsearch request + */ + public FilterAggregationBuilder buildTopAggregation(String topAggregationName, TopAggregationDefinition topAggregation, + Consumer extraFilters, Consumer subAggregations) { + BoolQueryBuilder filter = filterComputer.getTopAggregationFilter(topAggregation) + .orElseGet(QueryBuilders::boolQuery); + // optionally add extra filter(s) + extraFilters.accept(filter); + + FilterAggregationBuilder res = AggregationBuilders.filter(topAggregationName, filter); + subAggregations.accept(res); + checkState( + !res.getSubAggregations().isEmpty(), + "no sub-aggregation has been added to top-aggregation %s", topAggregationName); + return res; + } + + /** + * Same as {@link #buildTopAggregation(String, TopAggregationDefinition, Consumer, Consumer)} with built-in addition of a + * top-term sub aggregation based field defined by {@link TermTopAggregationDef#getFieldName()}. + */ + public FilterAggregationBuilder buildTermTopAggregation(String topAggregationName, TermTopAggregationDef topAggregation, + Consumer extraFilters, Consumer otherSubAggregations) { + Consumer subAggregations = t -> { + t.subAggregation(subAggregationHelper.buildTermsAggregation(topAggregationName, topAggregation)); + otherSubAggregations.accept(t); + }; + return buildTopAggregation(topAggregationName, topAggregation, extraFilters, subAggregations); + } + + public SubAggregationHelper getSubAggregationHelper() { + return subAggregationHelper; + } +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/package-info.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/package-info.java new file mode 100644 index 00000000000..498a7c7cc94 --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/searchrequest/package-info.java @@ -0,0 +1,23 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.server.es.searchrequest; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/AllFiltersTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/AllFiltersTest.java new file mode 100644 index 00000000000..e37eff78b2b --- /dev/null +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/AllFiltersTest.java @@ -0,0 +1,119 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.searchrequest; + +import java.util.List; +import java.util.Random; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.junit.Test; + +import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; + +public class AllFiltersTest { + @Test + public void newalways_returns_a_new_instance() { + int expected = 1 + new Random().nextInt(200); + RequestFiltersComputer.AllFilters[] instances = IntStream.range(0, expected) + .mapToObj(t -> RequestFiltersComputer.newAllFilters()) + .toArray(RequestFiltersComputer.AllFilters[]::new); + + assertThat(instances).hasSize(expected); + } + + @Test + public void addFilter_fails_if_name_is_null() { + String fieldName = randomAlphabetic(12); + RequestFiltersComputer.AllFilters allFilters = RequestFiltersComputer.newAllFilters(); + + Stream.of( + () -> allFilters.addFilter(null, boolQuery()), + () -> allFilters.addFilter(null, fieldName, boolQuery())) + .forEach(t -> assertThatThrownBy(t) + .isInstanceOf(NullPointerException.class) + .hasMessage("name can't be null")); + } + + @Test + public void addFilter_fails_if_fieldname_is_null() { + String name = randomAlphabetic(12); + RequestFiltersComputer.AllFilters allFilters = RequestFiltersComputer.newAllFilters(); + + assertThatThrownBy(() -> allFilters.addFilter(name, null, boolQuery())) + .isInstanceOf(NullPointerException.class) + .hasMessage("fieldName can't be null"); + } + + @Test + public void addFilter_fails_if_field_with_name_already_exists() { + String name1 = randomAlphabetic(12); + String name2 = randomAlphabetic(15); + String fieldName = randomAlphabetic(16); + String fieldName2 = randomAlphabetic(18); + RequestFiltersComputer.AllFilters allFilters = RequestFiltersComputer.newAllFilters(); + allFilters.addFilter(name1, boolQuery()); + allFilters.addFilter(name2, fieldName, boolQuery()); + + Stream.of( + // exact same call + () -> allFilters.addFilter(name1, boolQuery()), + // call with a different fieldName + () -> allFilters.addFilter(name1, fieldName, boolQuery())) + .forEach(t -> assertThatThrownBy(t) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("A filter with name " + name1 + " has already been added")); + Stream.of( + // exact same call + () -> allFilters.addFilter(name2, fieldName, boolQuery()), + // call with a different fieldName + () -> allFilters.addFilter(name2, fieldName2, boolQuery())) + .forEach(t -> assertThatThrownBy(t) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("A filter with name " + name2 + " has already been added")); + } + + @Test + public void addFilter_does_not_add_filter_if_QueryBuilder_is_null() { + String name = randomAlphabetic(12); + String name2 = randomAlphabetic(14); + RequestFiltersComputer.AllFilters allFilters = RequestFiltersComputer.newAllFilters(); + BoolQueryBuilder query = boolQuery(); + allFilters.addFilter(name, query) + .addFilter(name2, null); + + List all = allFilters.stream().collect(Collectors.toList()); + assertThat(all).hasSize(1); + assertThat(all.iterator().next()).isSameAs(query); + } + + @Test + public void stream_is_empty_when_addFilter_never_called() { + RequestFiltersComputer.AllFilters allFilters = RequestFiltersComputer.newAllFilters(); + + assertThat(allFilters.stream()).isEmpty(); + } +} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/RequestFiltersComputerTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/RequestFiltersComputerTest.java new file mode 100644 index 00000000000..b71b8919c67 --- /dev/null +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/RequestFiltersComputerTest.java @@ -0,0 +1,276 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.searchrequest; + +import com.google.common.collect.ImmutableSet; +import java.util.Arrays; +import java.util.Collections; +import java.util.Random; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.junit.Test; +import org.mockito.Mockito; +import org.sonar.server.es.searchrequest.RequestFiltersComputer.AllFilters; + +import static java.util.stream.Collectors.toSet; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.sonar.server.es.searchrequest.RequestFiltersComputer.newAllFilters; + +public class RequestFiltersComputerTest { + + private static final Random RANDOM = new Random(); + + @Test + public void getTopAggregationFilters_fails_with_IAE_when_no_TopAggregation_provided_in_constructor() { + RequestFiltersComputer underTest = new RequestFiltersComputer(newAllFilters(), Collections.emptySet()); + + assertThatThrownBy(() -> underTest.getTopAggregationFilter(Mockito.mock(TopAggregationDefinition.class))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("topAggregation must have been declared in constructor"); + } + + @Test + public void getTopAggregationFilters_fails_with_IAE_when_TopAggregation_was_not_provided_in_constructor() { + Set atLeastOneTopAggs = randomNonEmptyTopAggregations(RANDOM::nextBoolean); + RequestFiltersComputer underTest = new RequestFiltersComputer(newAllFilters(), atLeastOneTopAggs); + + atLeastOneTopAggs.forEach(underTest::getTopAggregationFilter); + assertThatThrownBy(() -> underTest.getTopAggregationFilter(Mockito.mock(TopAggregationDefinition.class))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("topAggregation must have been declared in constructor"); + } + + @Test + public void getQueryFilters_returns_empty_if_AllFilters_is_empty() { + Set atLeastOneTopAggs = randomNonEmptyTopAggregations(RANDOM::nextBoolean); + RequestFiltersComputer underTest = new RequestFiltersComputer(newAllFilters(), atLeastOneTopAggs); + + assertThat(underTest.getQueryFilters()).isEmpty(); + } + + @Test + public void getPostFilters_returns_empty_if_AllFilters_is_empty() { + Set atLeastOneTopAggs = randomNonEmptyTopAggregations(RANDOM::nextBoolean); + RequestFiltersComputer underTest = new RequestFiltersComputer(newAllFilters(), atLeastOneTopAggs); + + assertThat(underTest.getPostFilters()).isEmpty(); + } + + @Test + public void getTopAggregationFilter_returns_empty_if_AllFilters_is_empty() { + Set atLeastOneTopAggs = randomNonEmptyTopAggregations(RANDOM::nextBoolean); + RequestFiltersComputer underTest = new RequestFiltersComputer(newAllFilters(), atLeastOneTopAggs); + + atLeastOneTopAggs.forEach(topAgg -> assertThat(underTest.getTopAggregationFilter(topAgg)).isEmpty()); + } + + @Test + public void getQueryFilters_contains_all_filters_when_no_declared_topAggregation() { + AllFilters allFilters = randomNonEmptyAllFilters(); + RequestFiltersComputer underTest = new RequestFiltersComputer(allFilters, Collections.emptySet()); + + assertThat(underTest.getQueryFilters().get()).isEqualTo(toBoolQuery(allFilters.stream())); + } + + @Test + public void getPostFilters_returns_empty_when_no_declared_topAggregation() { + AllFilters allFilters = randomNonEmptyAllFilters(); + RequestFiltersComputer underTest = new RequestFiltersComputer(allFilters, Collections.emptySet()); + + assertThat(underTest.getPostFilters()).isEmpty(); + } + + @Test + public void getQueryFilters_contains_all_filters_when_no_declared_sticky_topAggregation() { + AllFilters allFilters = randomNonEmptyAllFilters(); + Set atLeastOneNonStickyTopAggs = randomNonEmptyTopAggregations(() -> false); + RequestFiltersComputer underTest = new RequestFiltersComputer(allFilters, atLeastOneNonStickyTopAggs); + + assertThat(underTest.getQueryFilters().get()).isEqualTo(toBoolQuery(allFilters.stream())); + } + + @Test + public void getPostFilters_returns_empty_when_no_declared_sticky_topAggregation() { + AllFilters allFilters = randomNonEmptyAllFilters(); + Set atLeastOneNonStickyTopAggs = randomNonEmptyTopAggregations(() -> false); + RequestFiltersComputer underTest = new RequestFiltersComputer(allFilters, atLeastOneNonStickyTopAggs); + + assertThat(underTest.getPostFilters()).isEmpty(); + } + + @Test + public void getTopAggregationFilters_return_empty_when_no_declared_sticky_topAggregation() { + AllFilters allFilters = randomNonEmptyAllFilters(); + Set atLeastOneNonStickyTopAggs = randomNonEmptyTopAggregations(() -> false); + RequestFiltersComputer underTest = new RequestFiltersComputer(allFilters, atLeastOneNonStickyTopAggs); + + atLeastOneNonStickyTopAggs.forEach(topAgg -> assertThat(underTest.getTopAggregationFilter(topAgg)).isEmpty()); + } + + @Test + public void filters_on_field_of_sticky_TopAggregation_go_to_PostFilters_and_TopAgg_Filters_on_other_fields() { + AllFilters allFilters = newAllFilters(); + // has topAggs and two filters + String field1 = "field1"; + TopAggregationDefinition stickyTopAggField1 = new TopAggregationDef(field1, true); + TopAggregationDefinition nonStickyTopAggField1 = new TopAggregationDef(field1, false); + QueryBuilder filterField1_1 = newQuery(); + QueryBuilder filterField1_2 = newQuery(); + allFilters.addFilter(field1, filterField1_1); + allFilters.addFilter(field1 + "_2", field1, filterField1_2); + // has topAggs and one filter + String field2 = "field2"; + TopAggregationDefinition stickyTopAggField2 = new TopAggregationDef(field2, true); + TopAggregationDefinition nonStickyTopAggField2 = new TopAggregationDef(field2, false); + QueryBuilder filterField2 = newQuery(); + allFilters.addFilter(field2, filterField2); + // has only non-sticky top-agg and one filter + String field3 = "field3"; + TopAggregationDefinition nonStickyTopAggField3 = new TopAggregationDef(field3, false); + QueryBuilder filterField3 = newQuery(); + allFilters.addFilter(field3, filterField3); + // has one filter but no top agg + String field4 = "field4"; + QueryBuilder filterField4 = newQuery(); + allFilters.addFilter(field4, filterField4); + // has top-aggs by no filter + String field5 = "field5"; + TopAggregationDefinition stickyTopAggField5 = new TopAggregationDef(field5, true); + TopAggregationDefinition nonStickyTopAggField5 = new TopAggregationDef(field5, false); + Set declaredTopAggregations = ImmutableSet.of( + stickyTopAggField1, nonStickyTopAggField1, + stickyTopAggField2, nonStickyTopAggField2, + nonStickyTopAggField3, + stickyTopAggField5, nonStickyTopAggField5); + + RequestFiltersComputer underTest = new RequestFiltersComputer(allFilters, declaredTopAggregations); + + assertThat(underTest.getQueryFilters().get()).isEqualTo(toBoolQuery(filterField3, filterField4)); + BoolQueryBuilder postFilterQuery = toBoolQuery(filterField1_1, filterField1_2, filterField2); + assertThat(underTest.getPostFilters().get()).isEqualTo(postFilterQuery); + assertThat(underTest.getTopAggregationFilter(stickyTopAggField1).get()).isEqualTo(toBoolQuery(filterField2)); + assertThat(underTest.getTopAggregationFilter(nonStickyTopAggField1).get()).isEqualTo(postFilterQuery); + assertThat(underTest.getTopAggregationFilter(stickyTopAggField2).get()).isEqualTo(toBoolQuery(filterField1_1, filterField1_2)); + assertThat(underTest.getTopAggregationFilter(nonStickyTopAggField2).get()).isEqualTo(postFilterQuery); + assertThat(underTest.getTopAggregationFilter(nonStickyTopAggField3).get()).isEqualTo(postFilterQuery); + assertThat(underTest.getTopAggregationFilter(stickyTopAggField5).get()).isEqualTo(postFilterQuery); + assertThat(underTest.getTopAggregationFilter(nonStickyTopAggField5).get()).isEqualTo(postFilterQuery); + } + + @Test + public void getTopAggregationFilters_returns_empty_on_sticky_TopAgg_when_no_other_sticky_TopAgg() { + AllFilters allFilters = newAllFilters(); + // has topAggs and two filters + String field1 = "field1"; + TopAggregationDefinition stickyTopAggField1 = new TopAggregationDef(field1, true); + TopAggregationDefinition nonStickyTopAggField1 = new TopAggregationDef(field1, false); + QueryBuilder filterField1_1 = newQuery(); + QueryBuilder filterField1_2 = newQuery(); + allFilters.addFilter(field1, filterField1_1); + allFilters.addFilter(field1 + "_2", field1, filterField1_2); + // has only non-sticky top-agg and one filter + String field2 = "field2"; + TopAggregationDefinition nonStickyTopAggField2 = new TopAggregationDef(field2, false); + QueryBuilder filterField2 = newQuery(); + allFilters.addFilter(field2, filterField2); + Set declaredTopAggregations = ImmutableSet.of( + stickyTopAggField1, nonStickyTopAggField1, + nonStickyTopAggField2); + + RequestFiltersComputer underTest = new RequestFiltersComputer(allFilters, declaredTopAggregations); + + assertThat(underTest.getQueryFilters().get()).isEqualTo(toBoolQuery(filterField2)); + BoolQueryBuilder postFilterQuery = toBoolQuery(filterField1_1, filterField1_2); + assertThat(underTest.getPostFilters().get()).isEqualTo(postFilterQuery); + assertThat(underTest.getTopAggregationFilter(stickyTopAggField1)).isEmpty(); + assertThat(underTest.getTopAggregationFilter(nonStickyTopAggField1).get()).isEqualTo(postFilterQuery); + assertThat(underTest.getTopAggregationFilter(nonStickyTopAggField2).get()).isEqualTo(postFilterQuery); + } + + @Test + public void getQueryFilters_returns_empty_when_all_filters_have_sticky_TopAggs() { + AllFilters allFilters = newAllFilters(); + // has topAggs and two filters + String field1 = "field1"; + TopAggregationDefinition stickyTopAggField1 = new TopAggregationDef(field1, true); + TopAggregationDefinition nonStickyTopAggField1 = new TopAggregationDef(field1, false); + QueryBuilder filterField1_1 = newQuery(); + QueryBuilder filterField1_2 = newQuery(); + allFilters.addFilter(field1, filterField1_1); + allFilters.addFilter(field1 + "_2", field1, filterField1_2); + // has only sticky top-agg and one filter + String field2 = "field2"; + TopAggregationDefinition stickyTopAggField2 = new TopAggregationDef(field2, true); + QueryBuilder filterField2 = newQuery(); + allFilters.addFilter(field2, filterField2); + Set declaredTopAggregations = ImmutableSet.of( + stickyTopAggField1, nonStickyTopAggField1, + stickyTopAggField2); + + RequestFiltersComputer underTest = new RequestFiltersComputer(allFilters, declaredTopAggregations); + + assertThat(underTest.getQueryFilters()).isEmpty(); + BoolQueryBuilder postFilterQuery = toBoolQuery(filterField1_1, filterField1_2, filterField2); + assertThat(underTest.getPostFilters().get()).isEqualTo(postFilterQuery); + assertThat(underTest.getTopAggregationFilter(stickyTopAggField1).get()).isEqualTo(toBoolQuery(filterField2)); + assertThat(underTest.getTopAggregationFilter(nonStickyTopAggField1).get()).isEqualTo(postFilterQuery); + assertThat(underTest.getTopAggregationFilter(stickyTopAggField2).get()).isEqualTo(toBoolQuery(filterField1_1, filterField1_2)); + } + + private static Set randomNonEmptyTopAggregations(Supplier isSticky) { + return IntStream.range(0, 1 + RANDOM.nextInt(20)) + .mapToObj(i -> new TopAggregationDef("field_" + i, isSticky.get())) + .collect(toSet()); + } + + private static BoolQueryBuilder toBoolQuery(QueryBuilder first, QueryBuilder... others) { + return toBoolQuery(Stream.concat( + Stream.of(first), Arrays.stream(others))); + } + + private static BoolQueryBuilder toBoolQuery(Stream stream) { + BoolQueryBuilder res = boolQuery(); + stream.forEach(res::must); + return res; + } + + private static AllFilters randomNonEmptyAllFilters() { + AllFilters res = newAllFilters(); + IntStream.range(0, 1 + RANDOM.nextInt(22)) + .forEach(i -> res.addFilter("field_" + i, newQuery())); + return res; + } + + private static int queryCounter = 0; + + /** + * Creates unique queries + */ + private static QueryBuilder newQuery() { + return QueryBuilders.termQuery("query_" + (queryCounter++), "foo"); + } +} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/SubAggregationHelperTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/SubAggregationHelperTest.java new file mode 100644 index 00000000000..3644c7b3d1f --- /dev/null +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/SubAggregationHelperTest.java @@ -0,0 +1,164 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.searchrequest; + +import java.util.Random; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.BucketOrder; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.junit.Test; + +import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; +import static org.assertj.core.api.Assertions.assertThat; +import static org.sonar.server.es.searchrequest.TopAggregationHelperTest.DEFAULT_BUCKET_SIZE; + +public class SubAggregationHelperTest { + private static final BucketOrder ES_BUILTIN_TIE_BREAKER = BucketOrder.key(true); + private static final BucketOrder SQ_DEFAULT_BUCKET_ORDER = BucketOrder.count(false); + + private AbstractAggregationBuilder customSubAgg = AggregationBuilders.sum("foo"); + private SubAggregationHelper underTest = new SubAggregationHelper(); + private BucketOrder customOrder = BucketOrder.count(true); + private SubAggregationHelper underTestWithCustomSubAgg = new SubAggregationHelper(customSubAgg); + private SubAggregationHelper underTestWithCustomsSubAggAndOrder = new SubAggregationHelper(customSubAgg, customOrder); + + @Test + public void buildTermsAggregation_adds_term_subaggregation_with_minDoc_1_and_default_sort() { + String aggName = randomAlphabetic(10); + TermTopAggregationDef topAggregation = new TermTopAggregationDef("bar", false, null); + + Stream.of( + underTest, + underTestWithCustomSubAgg) + .forEach(t -> { + TermsAggregationBuilder agg = t.buildTermsAggregation(aggName, topAggregation); + + assertThat(agg.getName()).isEqualTo(aggName); + assertThat(agg.field()).isEqualTo(topAggregation.getFieldName()); + assertThat(agg.size()).isEqualTo(DEFAULT_BUCKET_SIZE); + assertThat(agg.minDocCount()).isEqualTo(1); + assertThat(agg.order()).isEqualTo(BucketOrder.compound(SQ_DEFAULT_BUCKET_ORDER, ES_BUILTIN_TIE_BREAKER)); + }); + } + + @Test + public void buildTermsAggregation_adds_custom_order_from_constructor() { + String aggName = randomAlphabetic(10); + TermTopAggregationDef topAggregation = new TermTopAggregationDef("bar", false, null); + + TermsAggregationBuilder agg = underTestWithCustomsSubAggAndOrder.buildTermsAggregation(aggName, topAggregation); + + assertThat(agg.getName()).isEqualTo(aggName); + assertThat(agg.field()).isEqualTo(topAggregation.getFieldName()); + assertThat(agg.order()).isEqualTo(BucketOrder.compound(customOrder, ES_BUILTIN_TIE_BREAKER)); + } + + @Test + public void buildTermsAggregation_adds_custom_sub_agg_from_constructor() { + String aggName = randomAlphabetic(10); + TermTopAggregationDef topAggregation = new TermTopAggregationDef("bar", false, null); + + Stream.of( + underTestWithCustomSubAgg, + underTestWithCustomsSubAggAndOrder) + .forEach(t -> { + TermsAggregationBuilder agg = t.buildTermsAggregation(aggName, topAggregation); + + assertThat(agg.getName()).isEqualTo(aggName); + assertThat(agg.field()).isEqualTo(topAggregation.getFieldName()); + assertThat(agg.getSubAggregations()).hasSize(1); + assertThat(agg.getSubAggregations().iterator().next()).isSameAs(customSubAgg); + }); + } + + @Test + public void buildTermsAggregation_adds_custom_size_if_TermTopAggregation_specifies_one() { + String aggName = randomAlphabetic(10); + int customSize = 1 + new Random().nextInt(400); + TermTopAggregationDef topAggregation = new TermTopAggregationDef("bar", false, customSize); + + Stream.of( + underTest, + underTestWithCustomSubAgg, + underTestWithCustomsSubAggAndOrder) + .forEach(t -> { + TermsAggregationBuilder agg = t.buildTermsAggregation(aggName, topAggregation); + + assertThat(agg.getName()).isEqualTo(aggName); + assertThat(agg.field()).isEqualTo(topAggregation.getFieldName()); + assertThat(agg.size()).isEqualTo(customSize); + }); + } + + @Test + public void buildSelectedItemsAggregation_returns_empty_if_no_selected_item() { + String aggName = randomAlphabetic(10); + TermTopAggregationDef topAggregation = new TermTopAggregationDef("bar", false, null); + + Stream.of( + underTest, + underTestWithCustomSubAgg, + underTestWithCustomsSubAggAndOrder) + .forEach(t -> assertThat(t.buildSelectedItemsAggregation(aggName, topAggregation, new Object[0])).isEmpty()); + } + + @Test + public void buildSelectedItemsAggregation_does_not_add_custom_order_from_constructor() { + String aggName = randomAlphabetic(10); + TermTopAggregationDef topAggregation = new TermTopAggregationDef("bar", false, null); + String[] selected = randomNonEmptySelected(); + + TermsAggregationBuilder agg = underTestWithCustomsSubAggAndOrder.buildSelectedItemsAggregation(aggName, topAggregation, selected) + .get(); + + assertThat(agg.getName()).isEqualTo(aggName + "_selected"); + assertThat(agg.field()).isEqualTo(topAggregation.getFieldName()); + assertThat(agg.order()).isEqualTo(BucketOrder.compound(SQ_DEFAULT_BUCKET_ORDER, ES_BUILTIN_TIE_BREAKER)); + } + + @Test + public void buildSelectedItemsAggregation_adds_custom_sub_agg_from_constructor() { + String aggName = randomAlphabetic(10); + TermTopAggregationDef topAggregation = new TermTopAggregationDef("bar", false, null); + String[] selected = randomNonEmptySelected(); + + Stream.of( + underTestWithCustomSubAgg, + underTestWithCustomsSubAggAndOrder) + .forEach(t -> { + TermsAggregationBuilder agg = t.buildSelectedItemsAggregation(aggName, topAggregation, selected).get(); + + assertThat(agg.getName()).isEqualTo(aggName + "_selected"); + assertThat(agg.field()).isEqualTo(topAggregation.getFieldName()); + assertThat(agg.getSubAggregations()).hasSize(1); + assertThat(agg.getSubAggregations().iterator().next()).isSameAs(customSubAgg); + }); + } + + private static String[] randomNonEmptySelected() { + return IntStream.range(0, 1 + new Random().nextInt(22)) + .mapToObj(i -> "selected_" + i) + .toArray(String[]::new); + } + +} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/TermTopAggregationDefTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/TermTopAggregationDefTest.java new file mode 100644 index 00000000000..ada7704d6ef --- /dev/null +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/TermTopAggregationDefTest.java @@ -0,0 +1,89 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.searchrequest; + +import java.util.Random; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; +import static org.assertj.core.api.Assertions.assertThat; + +public class TermTopAggregationDefTest { + private static final Random RANDOM = new Random(); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void fieldName_cannot_be_null() { + boolean sticky = RANDOM.nextBoolean(); + int maxTerms = 10; + + expectedException.expect(NullPointerException.class); + expectedException.expectMessage("fieldName can't be null"); + + new TermTopAggregationDef(null, sticky, maxTerms); + } + + @Test + public void maxTerms_can_be_null() { + String fieldName = randomAlphabetic(12); + boolean sticky = RANDOM.nextBoolean(); + + TermTopAggregationDef underTest = new TermTopAggregationDef(fieldName, sticky, null); + assertThat(underTest.getMaxTerms()).isEmpty(); + } + + @Test + public void maxTerms_can_be_0() { + String fieldName = randomAlphabetic(12); + boolean sticky = RANDOM.nextBoolean(); + + TermTopAggregationDef underTest = new TermTopAggregationDef(fieldName, sticky, 0); + assertThat(underTest.getMaxTerms()).hasValue(0); + } + + @Test + public void maxTerms_cant_be_less_than_0() { + String fieldName = randomAlphabetic(12); + boolean sticky = RANDOM.nextBoolean(); + int negativeNumber = -1 - RANDOM.nextInt(200); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("maxTerms can't be < 0"); + + new TermTopAggregationDef(fieldName, sticky, negativeNumber); + } + + @Test + public void getters() { + String fieldName = randomAlphabetic(12); + boolean sticky = RANDOM.nextBoolean(); + int maxTerms = RANDOM.nextInt(299); + TermTopAggregationDef underTest = new TermTopAggregationDef(fieldName, sticky, maxTerms); + + assertThat(underTest.getFieldName()).isEqualTo(fieldName); + assertThat(underTest.isSticky()).isEqualTo(sticky); + assertThat(underTest.getMaxTerms()).hasValue(maxTerms); + } + +} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/TopAggregationDefTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/TopAggregationDefTest.java new file mode 100644 index 00000000000..3367bef998c --- /dev/null +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/TopAggregationDefTest.java @@ -0,0 +1,52 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.searchrequest; + +import java.util.Random; +import org.apache.commons.lang.RandomStringUtils; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TopAggregationDefTest { + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void fieldName_cannot_be_null() { + boolean sticky = new Random().nextBoolean(); + expectedException.expect(NullPointerException.class); + expectedException.expectMessage("fieldName can't be null"); + + new TopAggregationDef(null, sticky); + } + + @Test + public void getters() { + String fieldName = RandomStringUtils.randomAlphabetic(12); + boolean sticky = new Random().nextBoolean(); + TopAggregationDef underTest = new TopAggregationDef(fieldName, sticky); + + assertThat(underTest.getFieldName()).isEqualTo(fieldName); + assertThat(underTest.isSticky()).isEqualTo(sticky); + } +} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/TopAggregationHelperTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/TopAggregationHelperTest.java new file mode 100644 index 00000000000..b020b01d3f2 --- /dev/null +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/TopAggregationHelperTest.java @@ -0,0 +1,236 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.searchrequest; + +import java.util.Arrays; +import java.util.Optional; +import java.util.Random; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder; +import org.junit.Test; + +import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; +import static org.sonar.server.es.searchrequest.TopAggregationHelper.NO_EXTRA_FILTER; +import static org.sonar.server.es.searchrequest.TopAggregationHelper.NO_OTHER_SUBAGGREGATION; + +public class TopAggregationHelperTest { + + public static final int DEFAULT_BUCKET_SIZE = 10; + private RequestFiltersComputer filtersComputer = mock(RequestFiltersComputer.class); + private SubAggregationHelper subAggregationHelper = mock(SubAggregationHelper.class); + private TopAggregationHelper underTest = new TopAggregationHelper(filtersComputer, subAggregationHelper); + + @Test + public void buildTopAggregation_fails_with_ISE_if_no_subaggregation_added_by_lambda() { + String aggregationName = "name"; + TopAggregationDefinition topAggregation = new TopAggregationDef("bar", false); + + assertThatThrownBy(() -> underTest.buildTopAggregation(aggregationName, topAggregation, NO_EXTRA_FILTER, NO_OTHER_SUBAGGREGATION)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("no sub-aggregation has been added to top-aggregation " + aggregationName); + } + + @Test + public void buildTopAggregation_adds_subAggregation_from_lambda_parameter() { + TopAggregationDefinition topAggregation = new TopAggregationDef("bar", false); + AggregationBuilder[] subAggs = IntStream.range(0, 1 + new Random().nextInt(12)) + .mapToObj(i -> AggregationBuilders.min("subAgg_" + i)) + .toArray(AggregationBuilder[]::new); + String topAggregationName = randomAlphabetic(10); + + AggregationBuilder aggregationBuilder = underTest.buildTopAggregation(topAggregationName, topAggregation, + NO_EXTRA_FILTER, t -> Arrays.stream(subAggs).forEach(t::subAggregation)); + + assertThat(aggregationBuilder.getName()).isEqualTo(topAggregationName); + assertThat(aggregationBuilder.getSubAggregations()).hasSize(subAggs.length); + assertThat(aggregationBuilder.getSubAggregations()).containsExactlyInAnyOrder(subAggs); + } + + @Test + public void buildTopAggregation_adds_filter_from_FiltersComputer_for_TopAggregation() { + TopAggregationDefinition topAggregation = new TopAggregationDef("bar", false); + TopAggregationDefinition otherTopAggregation = new TopAggregationDef("acme", false); + BoolQueryBuilder computerFilter = boolQuery(); + BoolQueryBuilder otherFilter = boolQuery(); + when(filtersComputer.getTopAggregationFilter(topAggregation)).thenReturn(Optional.of(computerFilter)); + when(filtersComputer.getTopAggregationFilter(otherTopAggregation)).thenReturn(Optional.of(otherFilter)); + MinAggregationBuilder subAggregation = AggregationBuilders.min("donut"); + String topAggregationName = randomAlphabetic(10); + + FilterAggregationBuilder aggregationBuilder = underTest.buildTopAggregation(topAggregationName, topAggregation, + NO_EXTRA_FILTER, t -> t.subAggregation(subAggregation)); + + assertThat(aggregationBuilder.getName()).isEqualTo(topAggregationName); + assertThat(aggregationBuilder.getFilter()).isSameAs(computerFilter); + } + + @Test + public void buildTopAggregation_has_empty_filter_when_FiltersComputer_returns_empty_for_TopAggregation() { + TopAggregationDefinition topAggregation = new TopAggregationDef("bar", false); + TopAggregationDefinition otherTopAggregation = new TopAggregationDef("acme", false); + BoolQueryBuilder otherFilter = boolQuery(); + when(filtersComputer.getTopAggregationFilter(topAggregation)).thenReturn(Optional.empty()); + when(filtersComputer.getTopAggregationFilter(otherTopAggregation)).thenReturn(Optional.of(otherFilter)); + MinAggregationBuilder subAggregation = AggregationBuilders.min("donut"); + String topAggregationName = randomAlphabetic(10); + + FilterAggregationBuilder aggregationBuilder = underTest.buildTopAggregation(topAggregationName, topAggregation, + NO_EXTRA_FILTER, t -> t.subAggregation(subAggregation)); + + assertThat(aggregationBuilder.getName()).isEqualTo(topAggregationName); + assertThat(aggregationBuilder.getFilter()).isEqualTo(boolQuery()).isNotSameAs(otherFilter); + } + + @Test + public void buildTopAggregation_adds_filter_from_FiltersComputer_for_TopAggregation_and_extra_one() { + String topAggregationName = randomAlphabetic(10); + TopAggregationDefinition topAggregation = new TopAggregationDef("bar", false); + TopAggregationDefinition otherTopAggregation = new TopAggregationDef("acme", false); + BoolQueryBuilder computerFilter = boolQuery(); + BoolQueryBuilder otherFilter = boolQuery(); + BoolQueryBuilder extraFilter = boolQuery(); + when(filtersComputer.getTopAggregationFilter(topAggregation)).thenReturn(Optional.of(computerFilter)); + when(filtersComputer.getTopAggregationFilter(otherTopAggregation)).thenReturn(Optional.of(otherFilter)); + MinAggregationBuilder subAggregation = AggregationBuilders.min("donut"); + + FilterAggregationBuilder aggregationBuilder = underTest.buildTopAggregation(topAggregationName, topAggregation, + t -> t.must(extraFilter), t -> t.subAggregation(subAggregation)); + + assertThat(aggregationBuilder.getName()).isEqualTo(topAggregationName); + assertThat(aggregationBuilder.getFilter()).isEqualTo(computerFilter); + assertThat(((BoolQueryBuilder) aggregationBuilder.getFilter()).must()).containsExactly(extraFilter); + } + + @Test + public void buildTopAggregation_does_not_add_subaggregation_from_subAggregationHelper() { + TopAggregationDefinition topAggregation = new TopAggregationDef("bar", false); + when(filtersComputer.getTopAggregationFilter(topAggregation)).thenReturn(Optional.empty()); + MinAggregationBuilder subAggregation = AggregationBuilders.min("donut"); + String topAggregationName = randomAlphabetic(10); + + underTest.buildTopAggregation(topAggregationName, topAggregation, NO_EXTRA_FILTER, t -> t.subAggregation(subAggregation)); + + verifyZeroInteractions(subAggregationHelper); + } + + @Test + public void buildTermTopAggregation_adds_term_subaggregation_from_subAggregationHelper() { + String topAggregationName = randomAlphabetic(10); + TermTopAggregationDef topAggregation = new TermTopAggregationDef("bar", false, null); + TermsAggregationBuilder termSubAgg = AggregationBuilders.terms("foo"); + when(subAggregationHelper.buildTermsAggregation(topAggregationName, topAggregation)).thenReturn(termSubAgg); + + FilterAggregationBuilder aggregationBuilder = underTest.buildTermTopAggregation(topAggregationName, topAggregation, + NO_EXTRA_FILTER, NO_OTHER_SUBAGGREGATION); + + assertThat(aggregationBuilder.getName()).isEqualTo(topAggregationName); + assertThat(aggregationBuilder.getSubAggregations()).hasSize(1); + assertThat(aggregationBuilder.getSubAggregations().iterator().next()).isSameAs(termSubAgg); + } + + @Test + public void buildTermTopAggregation_adds_subAggregation_from_lambda_parameter() { + TermTopAggregationDef topAggregation = new TermTopAggregationDef("bar", false, null); + AggregationBuilder[] subAggs = IntStream.range(0, 1 + new Random().nextInt(12)) + .mapToObj(i -> AggregationBuilders.min("subAgg_" + i)) + .toArray(AggregationBuilder[]::new); + String topAggregationName = randomAlphabetic(10); + TermsAggregationBuilder termSubAgg = AggregationBuilders.terms("foo"); + when(subAggregationHelper.buildTermsAggregation(topAggregationName, topAggregation)).thenReturn(termSubAgg); + AggregationBuilder[] allSubAggs = Stream.concat(Arrays.stream(subAggs), Stream.of(termSubAgg)).toArray(AggregationBuilder[]::new); + + AggregationBuilder aggregationBuilder = underTest.buildTermTopAggregation(topAggregationName, topAggregation, + NO_EXTRA_FILTER, t -> Arrays.stream(subAggs).forEach(t::subAggregation)); + + assertThat(aggregationBuilder.getName()).isEqualTo(topAggregationName); + assertThat(aggregationBuilder.getSubAggregations()).hasSize(allSubAggs.length); + assertThat(aggregationBuilder.getSubAggregations()).containsExactlyInAnyOrder(allSubAggs); + } + + @Test + public void buildTermTopAggregation_adds_filter_from_FiltersComputer_for_TopAggregation() { + TermTopAggregationDef topAggregation = new TermTopAggregationDef("bar", false, null); + TermTopAggregationDef otherTopAggregation = new TermTopAggregationDef("acme", false, null); + BoolQueryBuilder computerFilter = boolQuery(); + BoolQueryBuilder otherFilter = boolQuery(); + when(filtersComputer.getTopAggregationFilter(topAggregation)).thenReturn(Optional.of(computerFilter)); + when(filtersComputer.getTopAggregationFilter(otherTopAggregation)).thenReturn(Optional.of(otherFilter)); + String topAggregationName = randomAlphabetic(10); + TermsAggregationBuilder termSubAgg = AggregationBuilders.terms("foo"); + when(subAggregationHelper.buildTermsAggregation(topAggregationName, topAggregation)).thenReturn(termSubAgg); + + FilterAggregationBuilder aggregationBuilder = underTest.buildTermTopAggregation(topAggregationName, topAggregation, + NO_EXTRA_FILTER, NO_OTHER_SUBAGGREGATION); + + assertThat(aggregationBuilder.getName()).isEqualTo(topAggregationName); + assertThat(aggregationBuilder.getFilter()).isSameAs(computerFilter); + } + + @Test + public void buildTermTopAggregation_has_empty_filter_when_FiltersComputer_returns_empty_for_TopAggregation() { + TermTopAggregationDef topAggregation = new TermTopAggregationDef("bar", false, null); + TermTopAggregationDef otherTopAggregation = new TermTopAggregationDef("acme", false, null); + BoolQueryBuilder otherFilter = boolQuery(); + when(filtersComputer.getTopAggregationFilter(topAggregation)).thenReturn(Optional.empty()); + when(filtersComputer.getTopAggregationFilter(otherTopAggregation)).thenReturn(Optional.of(otherFilter)); + String topAggregationName = randomAlphabetic(10); + TermsAggregationBuilder termSubAgg = AggregationBuilders.terms("foo"); + when(subAggregationHelper.buildTermsAggregation(topAggregationName, topAggregation)).thenReturn(termSubAgg); + + FilterAggregationBuilder aggregationBuilder = underTest.buildTermTopAggregation(topAggregationName, topAggregation, + NO_EXTRA_FILTER, NO_OTHER_SUBAGGREGATION); + + assertThat(aggregationBuilder.getName()).isEqualTo(topAggregationName); + assertThat(aggregationBuilder.getFilter()).isEqualTo(boolQuery()).isNotSameAs(otherFilter); + } + + @Test + public void buildTermTopAggregation_adds_filter_from_FiltersComputer_for_TopAggregation_and_extra_one() { + String topAggregationName = randomAlphabetic(10); + TermTopAggregationDef topAggregation = new TermTopAggregationDef("bar", false, null); + TermTopAggregationDef otherTopAggregation = new TermTopAggregationDef("acme", false, null); + BoolQueryBuilder computerFilter = boolQuery(); + BoolQueryBuilder otherFilter = boolQuery(); + BoolQueryBuilder extraFilter = boolQuery(); + when(filtersComputer.getTopAggregationFilter(topAggregation)).thenReturn(Optional.of(computerFilter)); + when(filtersComputer.getTopAggregationFilter(otherTopAggregation)).thenReturn(Optional.of(otherFilter)); + TermsAggregationBuilder termSubAgg = AggregationBuilders.terms("foo"); + when(subAggregationHelper.buildTermsAggregation(topAggregationName, topAggregation)).thenReturn(termSubAgg); + + FilterAggregationBuilder aggregationBuilder = underTest.buildTermTopAggregation(topAggregationName, topAggregation, + t -> t.must(extraFilter), NO_OTHER_SUBAGGREGATION); + + assertThat(aggregationBuilder.getName()).isEqualTo(topAggregationName); + assertThat(aggregationBuilder.getFilter()).isEqualTo(computerFilter); + assertThat(((BoolQueryBuilder) aggregationBuilder.getFilter()).must()).containsExactly(extraFilter); + } +} diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java index ef60ff462ba..3713c2a3642 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java @@ -20,13 +20,11 @@ package org.sonar.server.issue.index; import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -34,6 +32,7 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; +import java.util.function.Consumer; import java.util.stream.IntStream; import java.util.stream.Stream; import javax.annotation.CheckForNull; @@ -72,18 +71,24 @@ import org.sonar.api.rules.RuleType; import org.sonar.api.utils.DateUtils; import org.sonar.api.utils.System2; import org.sonar.core.util.stream.MoreCollectors; +import org.sonar.db.rule.RuleDefinitionDto; import org.sonar.server.es.BaseDoc; import org.sonar.server.es.EsClient; import org.sonar.server.es.EsUtils; import org.sonar.server.es.IndexType; import org.sonar.server.es.SearchOptions; import org.sonar.server.es.Sorting; -import org.sonar.server.es.StickyFacetBuilder; +import org.sonar.server.es.searchrequest.RequestFiltersComputer; +import org.sonar.server.es.searchrequest.RequestFiltersComputer.AllFilters; +import org.sonar.server.es.searchrequest.SubAggregationHelper; +import org.sonar.server.es.searchrequest.TermTopAggregationDef; +import org.sonar.server.es.searchrequest.TopAggregationDef; +import org.sonar.server.es.searchrequest.TopAggregationDefinition; +import org.sonar.server.es.searchrequest.TopAggregationHelper; import org.sonar.server.issue.index.IssueQuery.PeriodStart; import org.sonar.server.permission.index.AuthorizationDoc; import org.sonar.server.permission.index.WebAuthorizationTypeSupport; import org.sonar.server.security.SecurityStandards; -import org.sonar.server.security.SecurityStandards.SQCategory; import org.sonar.server.user.UserSession; import org.sonar.server.view.index.ViewIndexDefinition; @@ -103,6 +108,8 @@ import static org.sonar.core.util.stream.MoreCollectors.uniqueIndex; import static org.sonar.server.es.BaseDoc.epochMillisToEpochSeconds; import static org.sonar.server.es.EsUtils.escapeSpecialRegexChars; import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE; +import static org.sonar.server.es.searchrequest.TopAggregationHelper.NO_EXTRA_FILTER; +import static org.sonar.server.es.searchrequest.TopAggregationHelper.NO_OTHER_SUBAGGREGATION; import static org.sonar.server.issue.index.IssueIndex.Facet.ASSIGNED_TO_ME; import static org.sonar.server.issue.index.IssueIndex.Facet.ASSIGNEES; import static org.sonar.server.issue.index.IssueIndex.Facet.AUTHOR; @@ -211,37 +218,45 @@ public class IssueIndex { .filter(termQuery(FIELD_ISSUE_STATUS, Issue.STATUS_REVIEWED)) .filter(termQuery(FIELD_ISSUE_RESOLUTION, Issue.RESOLUTION_FIXED)); + private static final boolean STICKY = true; + private static final boolean NON_STICKY = false; + private static final Object[] NO_SELECTED_VALUES = {0}; + private static final TopAggregationDefinition EFFORT_TOP_AGGREGATION = new TopAggregationDef(FIELD_ISSUE_EFFORT, NON_STICKY); + public enum Facet { - SEVERITIES(PARAM_SEVERITIES, FIELD_ISSUE_SEVERITY, Severity.ALL.size()), - STATUSES(PARAM_STATUSES, FIELD_ISSUE_STATUS, Issue.STATUSES.size()), + SEVERITIES(PARAM_SEVERITIES, FIELD_ISSUE_SEVERITY, STICKY, Severity.ALL.size()), + STATUSES(PARAM_STATUSES, FIELD_ISSUE_STATUS, STICKY, Issue.STATUSES.size()), // Resolutions facet returns one more element than the number of resolutions to take into account unresolved issues - RESOLUTIONS(PARAM_RESOLUTIONS, FIELD_ISSUE_RESOLUTION, Issue.RESOLUTIONS.size() + 1), - TYPES(PARAM_TYPES, FIELD_ISSUE_TYPE, RuleType.values().length), - LANGUAGES(PARAM_LANGUAGES, FIELD_ISSUE_LANGUAGE, MAX_FACET_SIZE), - RULES(PARAM_RULES, FIELD_ISSUE_RULE_ID, MAX_FACET_SIZE), - TAGS(PARAM_TAGS, FIELD_ISSUE_TAGS, MAX_FACET_SIZE), - AUTHORS(DEPRECATED_PARAM_AUTHORS, FIELD_ISSUE_AUTHOR_LOGIN, MAX_FACET_SIZE), - AUTHOR(PARAM_AUTHOR, FIELD_ISSUE_AUTHOR_LOGIN, MAX_FACET_SIZE), - PROJECT_UUIDS(FACET_PROJECTS, FIELD_ISSUE_PROJECT_UUID, MAX_FACET_SIZE), - MODULE_UUIDS(PARAM_MODULE_UUIDS, FIELD_ISSUE_MODULE_UUID, MAX_FACET_SIZE), - FILE_UUIDS(PARAM_FILE_UUIDS, FIELD_ISSUE_COMPONENT_UUID, MAX_FACET_SIZE), - DIRECTORIES(PARAM_DIRECTORIES, FIELD_ISSUE_DIRECTORY_PATH, MAX_FACET_SIZE), - ASSIGNEES(PARAM_ASSIGNEES, FIELD_ISSUE_ASSIGNEE_UUID, MAX_FACET_SIZE), - ASSIGNED_TO_ME(FACET_ASSIGNED_TO_ME, FIELD_ISSUE_ASSIGNEE_UUID, 1), - OWASP_TOP_10(PARAM_OWASP_TOP_10, FIELD_ISSUE_OWASP_TOP_10, DEFAULT_FACET_SIZE), - SANS_TOP_25(PARAM_SANS_TOP_25, FIELD_ISSUE_SANS_TOP_25, DEFAULT_FACET_SIZE), - CWE(PARAM_CWE, FIELD_ISSUE_CWE, DEFAULT_FACET_SIZE), - CREATED_AT(PARAM_CREATED_AT, FIELD_ISSUE_FUNC_CREATED_AT, DEFAULT_FACET_SIZE), - SONARSOURCE_SECURITY(PARAM_SONARSOURCE_SECURITY, FIELD_ISSUE_SQ_SECURITY_CATEGORY, DEFAULT_FACET_SIZE); + RESOLUTIONS(PARAM_RESOLUTIONS, FIELD_ISSUE_RESOLUTION, STICKY, Issue.RESOLUTIONS.size() + 1), + TYPES(PARAM_TYPES, FIELD_ISSUE_TYPE, STICKY, RuleType.values().length), + LANGUAGES(PARAM_LANGUAGES, FIELD_ISSUE_LANGUAGE, STICKY, MAX_FACET_SIZE), + RULES(PARAM_RULES, FIELD_ISSUE_RULE_ID, STICKY, MAX_FACET_SIZE), + TAGS(PARAM_TAGS, FIELD_ISSUE_TAGS, STICKY, MAX_FACET_SIZE), + AUTHORS(DEPRECATED_PARAM_AUTHORS, FIELD_ISSUE_AUTHOR_LOGIN, STICKY, MAX_FACET_SIZE), + AUTHOR(PARAM_AUTHOR, FIELD_ISSUE_AUTHOR_LOGIN, STICKY, MAX_FACET_SIZE), + PROJECT_UUIDS(FACET_PROJECTS, FIELD_ISSUE_PROJECT_UUID, STICKY, MAX_FACET_SIZE), + MODULE_UUIDS(PARAM_MODULE_UUIDS, FIELD_ISSUE_MODULE_UUID, STICKY, MAX_FACET_SIZE), + FILE_UUIDS(PARAM_FILE_UUIDS, FIELD_ISSUE_COMPONENT_UUID, STICKY, MAX_FACET_SIZE), + DIRECTORIES(PARAM_DIRECTORIES, FIELD_ISSUE_DIRECTORY_PATH, STICKY, MAX_FACET_SIZE), + ASSIGNEES(PARAM_ASSIGNEES, FIELD_ISSUE_ASSIGNEE_UUID, STICKY, MAX_FACET_SIZE), + ASSIGNED_TO_ME(FACET_ASSIGNED_TO_ME, FIELD_ISSUE_ASSIGNEE_UUID, STICKY, 1), + OWASP_TOP_10(PARAM_OWASP_TOP_10, FIELD_ISSUE_OWASP_TOP_10, STICKY, DEFAULT_FACET_SIZE), + SANS_TOP_25(PARAM_SANS_TOP_25, FIELD_ISSUE_SANS_TOP_25, STICKY, DEFAULT_FACET_SIZE), + CWE(PARAM_CWE, FIELD_ISSUE_CWE, STICKY, DEFAULT_FACET_SIZE), + CREATED_AT(PARAM_CREATED_AT, FIELD_ISSUE_FUNC_CREATED_AT, NON_STICKY), + SONARSOURCE_SECURITY(PARAM_SONARSOURCE_SECURITY, FIELD_ISSUE_SQ_SECURITY_CATEGORY, STICKY, DEFAULT_FACET_SIZE); private final String name; - private final String fieldName; - private final int size; + private final TopAggregationDefinition topAggregation; + + Facet(String name, String fieldName, boolean sticky, int size) { + this.name = name; + this.topAggregation = new TermTopAggregationDef(fieldName, sticky, size); + } - Facet(String name, String fieldName, int size) { + Facet(String name, String fieldName, boolean sticky) { this.name = name; - this.fieldName = fieldName; - this.size = size; + this.topAggregation = new TopAggregationDef(fieldName, sticky); } public String getName() { @@ -249,14 +264,21 @@ public class IssueIndex { } public String getFieldName() { - return fieldName; + return topAggregation.getFieldName(); + } + + public TopAggregationDefinition getTopAggregationDef() { + return topAggregation; } - public int getSize() { - return size; + public TermTopAggregationDef getTermTopAggregationDef() { + return (TermTopAggregationDef) topAggregation; } } + private static final Map FACETS_BY_NAME = Arrays.stream(Facet.values()) + .collect(uniqueIndex(Facet::getName)); + private static final String SUBSTRING_MATCH_REGEXP = ".*%s.*"; // TODO to be documented // TODO move to Facets ? @@ -308,30 +330,39 @@ public class IssueIndex { } public SearchResponse search(IssueQuery query, SearchOptions options) { - SearchRequestBuilder requestBuilder = client.prepareSearch(TYPE_ISSUE.getMainType()); + SearchRequestBuilder esRequest = client.prepareSearch(TYPE_ISSUE.getMainType()); - configureSorting(query, requestBuilder); - configurePagination(options, requestBuilder); - configureRouting(query, options, requestBuilder); + configureSorting(query, esRequest); + configurePagination(options, esRequest); + configureRouting(query, options, esRequest); - QueryBuilder esQuery = matchAllQuery(); - BoolQueryBuilder esFilter = boolQuery(); - Map filters = createFilters(query); - for (QueryBuilder filter : filters.values()) { - if (filter != null) { - esFilter.must(filter); - } - } - if (esFilter.hasClauses()) { - requestBuilder.setQuery(boolQuery().must(esQuery).filter(esFilter)); - } else { - requestBuilder.setQuery(esQuery); - } + AllFilters allFilters = createAllFilters(query); + RequestFiltersComputer filterComputer = newFilterComputer(options, allFilters); + + configureTopAggregations(query, options, esRequest, allFilters, filterComputer); + configureQuery(esRequest, filterComputer); + configureTopFilters(esRequest, filterComputer); + + esRequest.setFetchSource(false); + + return esRequest.get(); + } + + private void configureTopAggregations(IssueQuery query, SearchOptions options, SearchRequestBuilder esRequest, AllFilters allFilters, RequestFiltersComputer filterComputer) { + TopAggregationHelper aggregationHelper = newAggregationHelper(filterComputer, query); - configureStickyFacets(query, options, filters, esQuery, requestBuilder); - requestBuilder.addAggregation(EFFORT_AGGREGATION); - requestBuilder.setFetchSource(false); - return requestBuilder.get(); + configureTopAggregations(aggregationHelper, query, options, allFilters, esRequest); + } + + private static void configureQuery(SearchRequestBuilder esRequest, RequestFiltersComputer filterComputer) { + QueryBuilder esQuery = filterComputer.getQueryFilters() + .map(t -> (QueryBuilder) boolQuery().must(matchAllQuery()).filter(t)) + .orElse(matchAllQuery()); + esRequest.setQuery(esQuery); + } + + private static void configureTopFilters(SearchRequestBuilder esRequest, RequestFiltersComputer filterComputer) { + filterComputer.getPostFilters().ifPresent(esRequest::setPostFilter); } /** @@ -352,44 +383,43 @@ public class IssueIndex { esSearch.setFrom(options.getOffset()).setSize(options.getLimit()); } - private Map createFilters(IssueQuery query) { - Map filters = new HashMap<>(); - filters.put("__indexType", termQuery(FIELD_INDEX_TYPE, TYPE_ISSUE.getName())); - filters.put("__authorization", createAuthorizationFilter()); + private AllFilters createAllFilters(IssueQuery query) { + AllFilters filters = RequestFiltersComputer.newAllFilters(); + filters.addFilter("__indexType", FIELD_INDEX_TYPE, termQuery(FIELD_INDEX_TYPE, TYPE_ISSUE.getName())); + filters.addFilter("__authorization", "parent", createAuthorizationFilter()); // Issue is assigned Filter if (BooleanUtils.isTrue(query.assigned())) { - filters.put(IS_ASSIGNED_FILTER, existsQuery(FIELD_ISSUE_ASSIGNEE_UUID)); + filters.addFilter(IS_ASSIGNED_FILTER, FIELD_ISSUE_ASSIGNEE_UUID, existsQuery(FIELD_ISSUE_ASSIGNEE_UUID)); } else if (BooleanUtils.isFalse(query.assigned())) { - filters.put(IS_ASSIGNED_FILTER, boolQuery().mustNot(existsQuery(FIELD_ISSUE_ASSIGNEE_UUID))); + filters.addFilter(IS_ASSIGNED_FILTER, FIELD_ISSUE_ASSIGNEE_UUID, boolQuery().mustNot(existsQuery(FIELD_ISSUE_ASSIGNEE_UUID))); } // Issue is Resolved Filter - String isResolved = "__isResolved"; if (BooleanUtils.isTrue(query.resolved())) { - filters.put(isResolved, existsQuery(FIELD_ISSUE_RESOLUTION)); + filters.addFilter("__isResolved", FIELD_ISSUE_RESOLUTION, existsQuery(FIELD_ISSUE_RESOLUTION)); } else if (BooleanUtils.isFalse(query.resolved())) { - filters.put(isResolved, boolQuery().mustNot(existsQuery(FIELD_ISSUE_RESOLUTION))); + filters.addFilter("__isResolved", FIELD_ISSUE_RESOLUTION, boolQuery().mustNot(existsQuery(FIELD_ISSUE_RESOLUTION))); } // Field Filters - filters.put(FIELD_ISSUE_KEY, createTermsFilter(FIELD_ISSUE_KEY, query.issueKeys())); - filters.put(FIELD_ISSUE_ASSIGNEE_UUID, createTermsFilter(FIELD_ISSUE_ASSIGNEE_UUID, query.assignees())); - filters.put(FIELD_ISSUE_LANGUAGE, createTermsFilter(FIELD_ISSUE_LANGUAGE, query.languages())); - filters.put(FIELD_ISSUE_TAGS, createTermsFilter(FIELD_ISSUE_TAGS, query.tags())); - filters.put(FIELD_ISSUE_TYPE, createTermsFilter(FIELD_ISSUE_TYPE, query.types())); - filters.put(FIELD_ISSUE_RESOLUTION, createTermsFilter(FIELD_ISSUE_RESOLUTION, query.resolutions())); - filters.put(FIELD_ISSUE_AUTHOR_LOGIN, createTermsFilter(FIELD_ISSUE_AUTHOR_LOGIN, query.authors())); - filters.put(FIELD_ISSUE_RULE_ID, createTermsFilter( + filters.addFilter(FIELD_ISSUE_KEY, createTermsFilter(FIELD_ISSUE_KEY, query.issueKeys())); + filters.addFilter(FIELD_ISSUE_ASSIGNEE_UUID, createTermsFilter(FIELD_ISSUE_ASSIGNEE_UUID, query.assignees())); + filters.addFilter(FIELD_ISSUE_LANGUAGE, createTermsFilter(FIELD_ISSUE_LANGUAGE, query.languages())); + filters.addFilter(FIELD_ISSUE_TAGS, createTermsFilter(FIELD_ISSUE_TAGS, query.tags())); + filters.addFilter(FIELD_ISSUE_TYPE, createTermsFilter(FIELD_ISSUE_TYPE, query.types())); + filters.addFilter(FIELD_ISSUE_RESOLUTION, createTermsFilter(FIELD_ISSUE_RESOLUTION, query.resolutions())); + filters.addFilter(FIELD_ISSUE_AUTHOR_LOGIN, createTermsFilter(FIELD_ISSUE_AUTHOR_LOGIN, query.authors())); + filters.addFilter(FIELD_ISSUE_RULE_ID, createTermsFilter( FIELD_ISSUE_RULE_ID, query.rules().stream().map(IssueDoc::formatRuleId).collect(toList()))); - filters.put(FIELD_ISSUE_STATUS, createTermsFilter(FIELD_ISSUE_STATUS, query.statuses())); - filters.put(FIELD_ISSUE_ORGANIZATION_UUID, createTermFilter(FIELD_ISSUE_ORGANIZATION_UUID, query.organizationUuid())); - filters.put(FIELD_ISSUE_OWASP_TOP_10, createTermsFilter(FIELD_ISSUE_OWASP_TOP_10, query.owaspTop10())); - filters.put(FIELD_ISSUE_SANS_TOP_25, createTermsFilter(FIELD_ISSUE_SANS_TOP_25, query.sansTop25())); - filters.put(FIELD_ISSUE_CWE, createTermsFilter(FIELD_ISSUE_CWE, query.cwe())); + filters.addFilter(FIELD_ISSUE_STATUS, createTermsFilter(FIELD_ISSUE_STATUS, query.statuses())); + filters.addFilter(FIELD_ISSUE_ORGANIZATION_UUID, createTermFilter(FIELD_ISSUE_ORGANIZATION_UUID, query.organizationUuid())); + filters.addFilter(FIELD_ISSUE_OWASP_TOP_10, createTermsFilter(FIELD_ISSUE_OWASP_TOP_10, query.owaspTop10())); + filters.addFilter(FIELD_ISSUE_SANS_TOP_25, createTermsFilter(FIELD_ISSUE_SANS_TOP_25, query.sansTop25())); + filters.addFilter(FIELD_ISSUE_CWE, createTermsFilter(FIELD_ISSUE_CWE, query.cwe())); addSeverityFilter(query, filters); - filters.put(FIELD_ISSUE_SQ_SECURITY_CATEGORY, createTermsFilter(FIELD_ISSUE_SQ_SECURITY_CATEGORY, query.sonarsourceSecurity())); + filters.addFilter(FIELD_ISSUE_SQ_SECURITY_CATEGORY, createTermsFilter(FIELD_ISSUE_SQ_SECURITY_CATEGORY, query.sonarsourceSecurity())); addComponentRelatedFilters(query, filters); addDatesFilter(filters, query); @@ -397,17 +427,17 @@ public class IssueIndex { return filters; } - private static void addSeverityFilter(IssueQuery query, Map filters) { + private static void addSeverityFilter(IssueQuery query, AllFilters allFilters) { QueryBuilder severityFieldFilter = createTermsFilter(FIELD_ISSUE_SEVERITY, query.severities()); if (severityFieldFilter != null) { - filters.put(FIELD_ISSUE_SEVERITY, boolQuery() + allFilters.addFilter(FIELD_ISSUE_SEVERITY, boolQuery() .must(severityFieldFilter) // Ignore severity of Security HotSpots .mustNot(termQuery(FIELD_ISSUE_TYPE, SECURITY_HOTSPOT.name()))); } } - private static void addComponentRelatedFilters(IssueQuery query, Map filters) { + private static void addComponentRelatedFilters(IssueQuery query, AllFilters filters) { addCommonComponentRelatedFilters(query, filters); if (query.viewUuids().isEmpty()) { addBranchComponentRelatedFilters(query, filters); @@ -416,35 +446,46 @@ public class IssueIndex { } } - private static void addCommonComponentRelatedFilters(IssueQuery query, Map filters) { + private static void addCommonComponentRelatedFilters(IssueQuery query, AllFilters filters) { QueryBuilder componentFilter = createTermsFilter(FIELD_ISSUE_COMPONENT_UUID, query.componentUuids()); - QueryBuilder projectFilter = createTermsFilter(FIELD_ISSUE_PROJECT_UUID, query.projectUuids()); - QueryBuilder moduleRootFilter = createTermsFilter(FIELD_ISSUE_MODULE_PATH, query.moduleRootUuids()); - QueryBuilder moduleFilter = createTermsFilter(FIELD_ISSUE_MODULE_UUID, query.moduleUuids()); - QueryBuilder directoryFilter = createTermsFilter(FIELD_ISSUE_DIRECTORY_PATH, query.directories()); QueryBuilder fileFilter = createTermsFilter(FIELD_ISSUE_COMPONENT_UUID, query.fileUuids()); if (BooleanUtils.isTrue(query.onComponentOnly())) { - filters.put(FIELD_ISSUE_COMPONENT_UUID, componentFilter); + filters.addFilter(FIELD_ISSUE_COMPONENT_UUID, componentFilter); } else { - filters.put(FIELD_ISSUE_PROJECT_UUID, projectFilter); - filters.put("__module", moduleRootFilter); - filters.put(FIELD_ISSUE_MODULE_UUID, moduleFilter); - filters.put(FIELD_ISSUE_DIRECTORY_PATH, directoryFilter); - filters.put(FIELD_ISSUE_COMPONENT_UUID, fileFilter != null ? fileFilter : componentFilter); + filters.addFilter( + FIELD_ISSUE_PROJECT_UUID, + createTermsFilter(FIELD_ISSUE_PROJECT_UUID, query.projectUuids())); + filters.addFilter( + "__module", + FIELD_ISSUE_MODULE_PATH, + createTermsFilter(FIELD_ISSUE_MODULE_PATH, query.moduleRootUuids())); + filters.addFilter( + FIELD_ISSUE_MODULE_UUID, + createTermsFilter(FIELD_ISSUE_MODULE_UUID, query.moduleUuids())); + filters.addFilter( + FIELD_ISSUE_DIRECTORY_PATH, + createTermsFilter(FIELD_ISSUE_DIRECTORY_PATH, query.directories())); + filters.addFilter( + FIELD_ISSUE_COMPONENT_UUID, + fileFilter == null ? componentFilter : fileFilter); } } - private static void addBranchComponentRelatedFilters(IssueQuery query, Map filters) { + private static void addBranchComponentRelatedFilters(IssueQuery query, AllFilters allFilters) { if (BooleanUtils.isTrue(query.onComponentOnly())) { return; } - QueryBuilder branchFilter = createTermFilter(FIELD_ISSUE_BRANCH_UUID, query.branchUuid()); - filters.put("__is_main_branch", createTermFilter(FIELD_ISSUE_IS_MAIN_BRANCH, Boolean.toString(query.isMainBranch()))); - filters.put(FIELD_ISSUE_BRANCH_UUID, branchFilter); + allFilters.addFilter( + "__is_main_branch", + FIELD_ISSUE_IS_MAIN_BRANCH, + createTermFilter(FIELD_ISSUE_IS_MAIN_BRANCH, Boolean.toString(query.isMainBranch()))); + allFilters.addFilter( + FIELD_ISSUE_BRANCH_UUID, + createTermFilter(FIELD_ISSUE_BRANCH_UUID, query.branchUuid())); } - private static void addViewRelatedFilters(IssueQuery query, Map filters) { + private static void addViewRelatedFilters(IssueQuery query, AllFilters allFilters) { if (BooleanUtils.isTrue(query.onComponentOnly())) { return; } @@ -452,10 +493,10 @@ public class IssueIndex { String branchUuid = query.branchUuid(); boolean onApplicationBranch = branchUuid != null && !viewUuids.isEmpty(); if (onApplicationBranch) { - filters.put("__view", createViewFilter(singletonList(query.branchUuid()))); + allFilters.addFilter("__view", createViewFilter(singletonList(query.branchUuid()))); } else { - filters.put("__is_main_branch", createTermFilter(FIELD_ISSUE_IS_MAIN_BRANCH, Boolean.toString(true))); - filters.put("__view", createViewFilter(viewUuids)); + allFilters.addFilter("__is_main_branch", createTermFilter(FIELD_ISSUE_IS_MAIN_BRANCH, Boolean.toString(true))); + allFilters.addFilter("__view", createViewFilter(viewUuids)); } } @@ -478,6 +519,26 @@ public class IssueIndex { return viewsFilter; } + private static RequestFiltersComputer newFilterComputer(SearchOptions options, AllFilters allFilters) { + Collection facetNames = options.getFacets(); + Set facets = Stream.concat( + Stream.of(EFFORT_TOP_AGGREGATION), + facetNames.stream() + .map(FACETS_BY_NAME::get) + .filter(Objects::nonNull) + .map(Facet::getTopAggregationDef)) + .collect(MoreCollectors.toSet(facetNames.size())); + + return new RequestFiltersComputer(allFilters, facets); + } + + private static TopAggregationHelper newAggregationHelper(RequestFiltersComputer filterComputer, IssueQuery query) { + if (hasQueryEffortFacet(query)) { + return new TopAggregationHelper(filterComputer, new SubAggregationHelper(EFFORT_AGGREGATION, EFFORT_AGGREGATION_ORDER)); + } + return new TopAggregationHelper(filterComputer, new SubAggregationHelper()); + } + private static AggregationBuilder addEffortAggregationIfNeeded(IssueQuery query, AggregationBuilder aggregation) { if (hasQueryEffortFacet(query)) { aggregation.subAggregation(EFFORT_AGGREGATION); @@ -489,69 +550,6 @@ public class IssueIndex { return FACET_MODE_EFFORT.equals(query.facetMode()); } - private static AggregationBuilder createSeverityFacet(IssueQuery query, Map filters, QueryBuilder queryBuilder) { - String fieldName = SEVERITIES.getFieldName(); - String facetName = SEVERITIES.getName(); - StickyFacetBuilder stickyFacetBuilder = newStickyFacetBuilder(query, filters, queryBuilder); - BoolQueryBuilder facetFilter = stickyFacetBuilder.getStickyFacetFilter(fieldName) - // Ignore severity of Security HotSpots - .mustNot(termQuery(FIELD_ISSUE_TYPE, SECURITY_HOTSPOT.name())); - FilterAggregationBuilder facetTopAggregation = stickyFacetBuilder.buildTopFacetAggregation(fieldName, facetName, facetFilter, SEVERITIES.getSize()); - return AggregationBuilders - .global(facetName) - .subAggregation(facetTopAggregation); - } - - private static AggregationBuilder createAssigneesFacet(IssueQuery query, Map filters, QueryBuilder queryBuilder) { - String fieldName = ASSIGNEES.getFieldName(); - String facetName = ASSIGNEES.getName(); - - // Same as in super.stickyFacetBuilder - Map assigneeFilters = new HashMap<>(filters); - assigneeFilters.remove(IS_ASSIGNED_FILTER); - assigneeFilters.remove(fieldName); - StickyFacetBuilder stickyFacetBuilder = newStickyFacetBuilder(query, assigneeFilters, queryBuilder); - BoolQueryBuilder facetFilter = stickyFacetBuilder.getStickyFacetFilter(fieldName); - FilterAggregationBuilder facetTopAggregation = stickyFacetBuilder.buildTopFacetAggregation(fieldName, facetName, facetFilter, ASSIGNEES.getSize()); - if (!query.assignees().isEmpty()) { - facetTopAggregation = stickyFacetBuilder.addSelectedItemsToFacet(fieldName, facetName, facetTopAggregation, t -> t, query.assignees().toArray()); - } - - // Add missing facet for unassigned issues - facetTopAggregation.subAggregation( - addEffortAggregationIfNeeded(query, AggregationBuilders - .missing(facetName + FACET_SUFFIX_MISSING) - .field(fieldName))); - - return AggregationBuilders - .global(facetName) - .subAggregation(facetTopAggregation); - } - - private static AggregationBuilder createResolutionFacet(IssueQuery query, Map filters, QueryBuilder esQuery) { - String fieldName = RESOLUTIONS.getFieldName(); - String facetName = RESOLUTIONS.getName(); - - // Same as in super.stickyFacetBuilder - Map resolutionFilters = Maps.newHashMap(filters); - resolutionFilters.remove("__isResolved"); - resolutionFilters.remove(fieldName); - StickyFacetBuilder stickyFacetBuilder = newStickyFacetBuilder(query, resolutionFilters, esQuery); - BoolQueryBuilder facetFilter = stickyFacetBuilder.getStickyFacetFilter(fieldName); - FilterAggregationBuilder facetTopAggregation = stickyFacetBuilder.buildTopFacetAggregation(fieldName, facetName, facetFilter, RESOLUTIONS.getSize()); - facetTopAggregation = stickyFacetBuilder.addSelectedItemsToFacet(fieldName, facetName, facetTopAggregation, t -> t); - - // Add missing facet for unresolved issues - facetTopAggregation.subAggregation( - addEffortAggregationIfNeeded(query, AggregationBuilders - .missing(facetName + FACET_SUFFIX_MISSING) - .field(fieldName))); - - return AggregationBuilders - .global(facetName) - .subAggregation(facetTopAggregation); - } - @CheckForNull private static QueryBuilder createTermsFilter(String field, Collection values) { return values.isEmpty() ? null : termsQuery(field, values); @@ -579,35 +577,44 @@ public class IssueIndex { return authorizationTypeSupport.createQueryFilter(); } - private void addDatesFilter(Map filters, IssueQuery query) { + private void addDatesFilter(AllFilters filters, IssueQuery query) { PeriodStart createdAfter = query.createdAfter(); Date createdBefore = query.createdBefore(); validateCreationDateBounds(createdBefore, createdAfter != null ? createdAfter.date() : null); if (createdAfter != null) { - filters.put("__createdAfter", QueryBuilders - .rangeQuery(FIELD_ISSUE_FUNC_CREATED_AT) - .from(BaseDoc.dateToEpochSeconds(createdAfter.date()), createdAfter.inclusive())); + filters.addFilter( + "__createdAfter", + FIELD_ISSUE_FUNC_CREATED_AT, + QueryBuilders + .rangeQuery(FIELD_ISSUE_FUNC_CREATED_AT) + .from(BaseDoc.dateToEpochSeconds(createdAfter.date()), createdAfter.inclusive())); } if (createdBefore != null) { - filters.put("__createdBefore", QueryBuilders - .rangeQuery(FIELD_ISSUE_FUNC_CREATED_AT) - .lt(BaseDoc.dateToEpochSeconds(createdBefore))); + filters.addFilter( + "__createdBefore", + FIELD_ISSUE_FUNC_CREATED_AT, + QueryBuilders + .rangeQuery(FIELD_ISSUE_FUNC_CREATED_AT) + .lt(BaseDoc.dateToEpochSeconds(createdBefore))); } Date createdAt = query.createdAt(); if (createdAt != null) { - filters.put("__createdAt", termQuery(FIELD_ISSUE_FUNC_CREATED_AT, BaseDoc.dateToEpochSeconds(createdAt))); + filters.addFilter( + "__createdAt", + FIELD_ISSUE_FUNC_CREATED_AT, + termQuery(FIELD_ISSUE_FUNC_CREATED_AT, BaseDoc.dateToEpochSeconds(createdAt))); } } - private static void addCreatedAfterByProjectsFilter(Map filters, IssueQuery query) { + private static void addCreatedAfterByProjectsFilter(AllFilters allFilters, IssueQuery query) { Map createdAfterByProjectUuids = query.createdAfterByProjectUuids(); BoolQueryBuilder boolQueryBuilder = boolQuery(); createdAfterByProjectUuids.forEach((projectUuid, createdAfterDate) -> boolQueryBuilder.should(boolQuery() .filter(termQuery(FIELD_ISSUE_PROJECT_UUID, projectUuid)) .filter(rangeQuery(FIELD_ISSUE_FUNC_CREATED_AT).from(BaseDoc.dateToEpochSeconds(createdAfterDate.date()), createdAfterDate.inclusive())))); - filters.put("createdAfterByProjectUuids", boolQueryBuilder); + allFilters.addFilter("createdAfterByProjectUuids", boolQueryBuilder); } private void validateCreationDateBounds(@Nullable Date createdBefore, @Nullable Date createdAfter) { @@ -617,46 +624,111 @@ public class IssueIndex { "Start bound cannot be larger or equal to end bound"); } - private void configureStickyFacets(IssueQuery query, SearchOptions options, Map filters, QueryBuilder esQuery, SearchRequestBuilder esSearch) { - if (!options.getFacets().isEmpty()) { - StickyFacetBuilder stickyFacetBuilder = newStickyFacetBuilder(query, filters, esQuery); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, STATUSES); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, PROJECT_UUIDS, query.projectUuids().toArray()); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, MODULE_UUIDS, query.moduleUuids().toArray()); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, DIRECTORIES, query.directories().toArray()); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, FILE_UUIDS, query.fileUuids().toArray()); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, LANGUAGES, query.languages().toArray()); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, RULES, query.rules().stream().map(IssueDoc::formatRuleId).toArray()); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, AUTHORS, query.authors().toArray()); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, AUTHOR, query.authors().toArray()); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, TAGS, query.tags().toArray()); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, TYPES, query.types().toArray()); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, OWASP_TOP_10, query.owaspTop10().toArray()); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, SANS_TOP_25, query.sansTop25().toArray()); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, CWE, query.cwe().toArray()); - if (options.getFacets().contains(PARAM_SEVERITIES)) { - esSearch.addAggregation(createSeverityFacet(query, filters, esQuery)); - } - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, SONARSOURCE_SECURITY, query.sonarsourceSecurity().toArray()); - if (options.getFacets().contains(PARAM_RESOLUTIONS)) { - esSearch.addAggregation(createResolutionFacet(query, filters, esQuery)); - } - if (options.getFacets().contains(PARAM_ASSIGNEES)) { - esSearch.addAggregation(createAssigneesFacet(query, filters, esQuery)); - } - if (options.getFacets().contains(PARAM_CREATED_AT)) { - getCreatedAtFacet(query, filters, esQuery).ifPresent(esSearch::addAggregation); - } - addAssignedToMeFacetIfNeeded(esSearch, options, query, filters, esQuery); + private void configureTopAggregations(TopAggregationHelper aggregationHelper, IssueQuery query, SearchOptions options, + AllFilters queryFilters, SearchRequestBuilder esRequest) { + addFacetIfNeeded(options, aggregationHelper, esRequest, STATUSES, NO_SELECTED_VALUES); + addFacetIfNeeded(options, aggregationHelper, esRequest, PROJECT_UUIDS, query.projectUuids().toArray()); + addFacetIfNeeded(options, aggregationHelper, esRequest, MODULE_UUIDS, query.moduleUuids().toArray()); + addFacetIfNeeded(options, aggregationHelper, esRequest, DIRECTORIES, query.directories().toArray()); + addFacetIfNeeded(options, aggregationHelper, esRequest, FILE_UUIDS, query.fileUuids().toArray()); + addFacetIfNeeded(options, aggregationHelper, esRequest, LANGUAGES, query.languages().toArray()); + addFacetIfNeeded(options, aggregationHelper, esRequest, RULES, query.rules().stream().map(RuleDefinitionDto::getId).toArray()); + addFacetIfNeeded(options, aggregationHelper, esRequest, AUTHORS, query.authors().toArray()); + addFacetIfNeeded(options, aggregationHelper, esRequest, AUTHOR, query.authors().toArray()); + addFacetIfNeeded(options, aggregationHelper, esRequest, TAGS, query.tags().toArray()); + addFacetIfNeeded(options, aggregationHelper, esRequest, TYPES, query.types().toArray()); + addFacetIfNeeded(options, aggregationHelper, esRequest, OWASP_TOP_10, query.owaspTop10().toArray()); + addFacetIfNeeded(options, aggregationHelper, esRequest, SANS_TOP_25, query.sansTop25().toArray()); + addFacetIfNeeded(options, aggregationHelper, esRequest, CWE, query.cwe().toArray()); + addFacetIfNeeded(options, aggregationHelper, esRequest, SONARSOURCE_SECURITY, query.sonarsourceSecurity().toArray()); + addSeverityFacetIfNeeded(options, aggregationHelper, esRequest); + addResolutionFacetIfNeeded(options, query, aggregationHelper, esRequest); + addAssigneesFacetIfNeeded(options, query, aggregationHelper, esRequest); + addCreatedAtFacetIfNeeded(options, query, aggregationHelper, queryFilters, esRequest); + addAssignedToMeFacetIfNeeded(options, aggregationHelper, esRequest); + addEffortTopAggregation(aggregationHelper, esRequest); + } + + private static void addFacetIfNeeded(SearchOptions options, TopAggregationHelper aggregationHelper, + SearchRequestBuilder esRequest, Facet facet, Object[] selectedValues) { + if (!options.getFacets().contains(facet.getName())) { + return; + } + + FilterAggregationBuilder topAggregation = aggregationHelper.buildTermTopAggregation( + facet.getName(), facet.getTermTopAggregationDef(), + NO_EXTRA_FILTER, + t -> aggregationHelper.getSubAggregationHelper().buildSelectedItemsAggregation(facet.getName(), facet.getTermTopAggregationDef(), selectedValues) + .ifPresent(t::subAggregation)); + esRequest.addAggregation(topAggregation); + } + + private static void addSeverityFacetIfNeeded(SearchOptions options, TopAggregationHelper aggregationHelper, SearchRequestBuilder esRequest) { + if (!options.getFacets().contains(PARAM_SEVERITIES)) { + return; } + + AggregationBuilder aggregation = aggregationHelper.buildTermTopAggregation( + SEVERITIES.getName(), SEVERITIES.getTermTopAggregationDef(), + // Ignore severity of Security HotSpots + filter -> filter.mustNot(termQuery(FIELD_ISSUE_TYPE, SECURITY_HOTSPOT.name())), + NO_OTHER_SUBAGGREGATION); + esRequest.addAggregation(aggregation); } - private Optional getCreatedAtFacet(IssueQuery query, Map filters, QueryBuilder esQuery) { + private static void addResolutionFacetIfNeeded(SearchOptions options, IssueQuery query, TopAggregationHelper aggregationHelper, SearchRequestBuilder esRequest) { + if (!options.getFacets().contains(PARAM_RESOLUTIONS)) { + return; + } + + AggregationBuilder aggregation = aggregationHelper.buildTermTopAggregation( + RESOLUTIONS.getName(), RESOLUTIONS.getTermTopAggregationDef(), + NO_EXTRA_FILTER, + t -> { + // add aggregation of type "missing" to return count of unresolved issues in the facet + t.subAggregation( + addEffortAggregationIfNeeded(query, AggregationBuilders + .missing(RESOLUTIONS.getName() + FACET_SUFFIX_MISSING) + .field(RESOLUTIONS.getFieldName()))); + }); + esRequest.addAggregation(aggregation); + } + + private static void addAssigneesFacetIfNeeded(SearchOptions options, IssueQuery query, TopAggregationHelper aggregationHelper, SearchRequestBuilder esRequest) { + if (!options.getFacets().contains(PARAM_ASSIGNEES)) { + return; + } + + Consumer assigneeAggregations = t -> { + // optional second aggregation to return the issue count for selected assignees (if any) + Object[] assignees = query.assignees().toArray(); + aggregationHelper.getSubAggregationHelper().buildSelectedItemsAggregation(ASSIGNEES.getName(), ASSIGNEES.getTopAggregationDef(), assignees) + .ifPresent(t::subAggregation); + + // third aggregation to always return the count of unassigned in the assignee facet + t.subAggregation(addEffortAggregationIfNeeded(query, AggregationBuilders + .missing(ASSIGNEES.getName() + FACET_SUFFIX_MISSING) + .field(ASSIGNEES.getFieldName()))); + }; + + AggregationBuilder aggregation = aggregationHelper.buildTermTopAggregation( + ASSIGNEES.getName(), ASSIGNEES.getTermTopAggregationDef(), NO_EXTRA_FILTER, assigneeAggregations); + esRequest.addAggregation(aggregation); + } + + private void addCreatedAtFacetIfNeeded(SearchOptions options, IssueQuery query, TopAggregationHelper aggregationHelper, AllFilters allFilters, + SearchRequestBuilder esRequest) { + if (options.getFacets().contains(PARAM_CREATED_AT)) { + getCreatedAtFacet(query, aggregationHelper, allFilters).ifPresent(esRequest::addAggregation); + } + } + + private Optional getCreatedAtFacet(IssueQuery query, TopAggregationHelper aggregationHelper, AllFilters allFilters) { long startTime; boolean startInclusive; PeriodStart createdAfter = query.createdAfter(); if (createdAfter == null) { - OptionalLong minDate = getMinCreatedAt(filters, esQuery); + OptionalLong minDate = getMinCreatedAt(allFilters); if (!minDate.isPresent()) { return Optional.empty(); } @@ -670,38 +742,50 @@ public class IssueIndex { long endTime = createdBefore == null ? system.now() : createdBefore.getTime(); Duration timeSpan = new Duration(startTime, endTime); - DateHistogramInterval bucketSize = DateHistogramInterval.YEAR; + DateHistogramInterval bucketSize = computeDateHistogramBucketSize(timeSpan); + + FilterAggregationBuilder topAggregation = aggregationHelper.buildTopAggregation( + CREATED_AT.getName(), + CREATED_AT.getTopAggregationDef(), + NO_EXTRA_FILTER, + t -> { + AggregationBuilder dateHistogram = AggregationBuilders.dateHistogram(CREATED_AT.getName()) + .field(CREATED_AT.getFieldName()) + .dateHistogramInterval(bucketSize) + .minDocCount(0L) + .format(DateUtils.DATETIME_FORMAT) + .timeZone(DateTimeZone.forOffsetMillis(system.getDefaultTimeZone().getRawOffset())) + // ES dateHistogram bounds are inclusive while createdBefore parameter is exclusive + .extendedBounds(new ExtendedBounds(startInclusive ? startTime : (startTime + 1), endTime - 1L)); + addEffortAggregationIfNeeded(query, dateHistogram); + t.subAggregation(dateHistogram); + }); + + return Optional.of(topAggregation); + } + + private static DateHistogramInterval computeDateHistogramBucketSize(Duration timeSpan) { if (timeSpan.isShorterThan(TWENTY_DAYS)) { - bucketSize = DateHistogramInterval.DAY; - } else if (timeSpan.isShorterThan(TWENTY_WEEKS)) { - bucketSize = DateHistogramInterval.WEEK; - } else if (timeSpan.isShorterThan(TWENTY_MONTHS)) { - bucketSize = DateHistogramInterval.MONTH; - } - - AggregationBuilder dateHistogram = AggregationBuilders.dateHistogram(CREATED_AT.getName()) - .field(CREATED_AT.getFieldName()) - .dateHistogramInterval(bucketSize) - .minDocCount(0L) - .format(DateUtils.DATETIME_FORMAT) - .timeZone(DateTimeZone.forOffsetMillis(system.getDefaultTimeZone().getRawOffset())) - // ES dateHistogram bounds are inclusive while createdBefore parameter is exclusive - .extendedBounds(new ExtendedBounds(startInclusive ? startTime : (startTime + 1), endTime - 1L)); - addEffortAggregationIfNeeded(query, dateHistogram); - return Optional.of(dateHistogram); - } - - private OptionalLong getMinCreatedAt(Map filters, QueryBuilder esQuery) { + return DateHistogramInterval.DAY; + } + if (timeSpan.isShorterThan(TWENTY_WEEKS)) { + return DateHistogramInterval.WEEK; + } + if (timeSpan.isShorterThan(TWENTY_MONTHS)) { + return DateHistogramInterval.MONTH; + } + return DateHistogramInterval.YEAR; + } + + private OptionalLong getMinCreatedAt(AllFilters filters) { String facetNameAndField = CREATED_AT.getFieldName(); SearchRequestBuilder esRequest = client .prepareSearch(TYPE_ISSUE.getMainType()) .setSize(0); BoolQueryBuilder esFilter = boolQuery(); - filters.values().stream().filter(Objects::nonNull).forEach(esFilter::must); + filters.stream().filter(Objects::nonNull).forEach(esFilter::must); if (esFilter.hasClauses()) { - esRequest.setQuery(QueryBuilders.boolQuery().must(esQuery).filter(esFilter)); - } else { - esRequest.setQuery(esQuery); + esRequest.setQuery(QueryBuilders.boolQuery().filter(esFilter)); } esRequest.addAggregation(AggregationBuilders.min(facetNameAndField).field(facetNameAndField)); Min minValue = esRequest.get().getAggregations().get(facetNameAndField); @@ -713,43 +797,29 @@ public class IssueIndex { return OptionalLong.of((long) actualValue); } - private void addAssignedToMeFacetIfNeeded(SearchRequestBuilder builder, SearchOptions options, IssueQuery query, Map filters, QueryBuilder queryBuilder) { + private void addAssignedToMeFacetIfNeeded(SearchOptions options, TopAggregationHelper aggregationHelper, SearchRequestBuilder esRequest) { String uuid = userSession.getUuid(); - - if (!options.getFacets().contains(ASSIGNED_TO_ME.getName()) || StringUtils.isEmpty(uuid)) { - return; - } - - String fieldName = ASSIGNED_TO_ME.getFieldName(); - String facetName = ASSIGNED_TO_ME.getName(); - - // Same as in super.stickyFacetBuilder - StickyFacetBuilder assignedToMeFacetBuilder = newStickyFacetBuilder(query, filters, queryBuilder); - BoolQueryBuilder facetFilter = assignedToMeFacetBuilder.getStickyFacetFilter(IS_ASSIGNED_FILTER, fieldName); - - FilterAggregationBuilder facetTopAggregation = AggregationBuilders - .filter(facetName + "__filter", facetFilter) - .subAggregation(addEffortAggregationIfNeeded(query, AggregationBuilders.terms(facetName + "__terms") - .field(fieldName) - .includeExclude(new IncludeExclude(escapeSpecialRegexChars(uuid), null)))); - - builder.addAggregation( - AggregationBuilders.global(facetName) - .subAggregation(facetTopAggregation)); - } - - private static StickyFacetBuilder newStickyFacetBuilder(IssueQuery query, Map filters, QueryBuilder esQuery) { - if (hasQueryEffortFacet(query)) { - return new StickyFacetBuilder(esQuery, filters, EFFORT_AGGREGATION, EFFORT_AGGREGATION_ORDER); + if (options.getFacets().contains(ASSIGNED_TO_ME.getName()) && !StringUtils.isEmpty(uuid)) { + AggregationBuilder aggregation = aggregationHelper.buildTermTopAggregation( + ASSIGNED_TO_ME.getName(), + ASSIGNED_TO_ME.getTermTopAggregationDef(), + NO_EXTRA_FILTER, + t -> { + // add sub-aggregation to return issue count for current user + aggregationHelper.getSubAggregationHelper().buildSelectedItemsAggregation(ASSIGNED_TO_ME.getName(), ASSIGNED_TO_ME.getTermTopAggregationDef(), new String[] {uuid}) + .ifPresent(t::subAggregation); + }); + esRequest.addAggregation(aggregation); } - return new StickyFacetBuilder(esQuery, filters); } - private static void addSimpleStickyFacetIfNeeded(SearchOptions options, StickyFacetBuilder stickyFacetBuilder, SearchRequestBuilder esSearch, - Facet facet, Object... selectedValues) { - if (options.getFacets().contains(facet.getName())) { - esSearch.addAggregation(stickyFacetBuilder.buildStickyFacet(facet.getFieldName(), facet.getName(), facet.getSize(), selectedValues)); - } + private static void addEffortTopAggregation(TopAggregationHelper aggregationHelper, SearchRequestBuilder esRequest) { + AggregationBuilder topAggregation = aggregationHelper.buildTopAggregation( + FACET_MODE_EFFORT, + EFFORT_TOP_AGGREGATION, + NO_EXTRA_FILTER, + t -> t.subAggregation(EFFORT_AGGREGATION)); + esRequest.addAggregation(topAggregation); } public List searchTags(IssueQuery query, @Nullable String textQuery, int size) { @@ -790,11 +860,9 @@ public class IssueIndex { private BoolQueryBuilder createBoolFilter(IssueQuery query) { BoolQueryBuilder boolQuery = boolQuery(); - for (QueryBuilder filter : createFilters(query).values()) { - if (filter != null) { - boolQuery.must(filter); - } - } + createAllFilters(query).stream() + .filter(Objects::nonNull) + .forEach(boolQuery::must); return boolQuery; } @@ -880,7 +948,7 @@ public class IssueIndex { public List getSonarSourceReport(String projectUuid, boolean isViewOrApp, boolean includeCwe) { SearchRequestBuilder request = prepareNonClosedVulnerabilitiesAndHotspotSearch(projectUuid, isViewOrApp); - Arrays.stream(SQCategory.values()) + Arrays.stream(SecurityStandards.SQCategory.values()) .forEach(sonarsourceCategory -> request.addAggregation( newSecurityReportSubAggregations( AggregationBuilders.filter(sonarsourceCategory.getKey(), boolQuery().filter(termQuery(FIELD_ISSUE_SQ_SECURITY_CATEGORY, sonarsourceCategory.getKey()))), -- 2.39.5