You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

ProjectMeasuresIndex.java 23KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457
  1. /*
  2. * SonarQube
  3. * Copyright (C) 2009-2019 SonarSource SA
  4. * mailto:info AT sonarsource DOT com
  5. *
  6. * This program is free software; you can redistribute it and/or
  7. * modify it under the terms of the GNU Lesser General Public
  8. * License as published by the Free Software Foundation; either
  9. * version 3 of the License, or (at your option) any later version.
  10. *
  11. * This program is distributed in the hope that it will be useful,
  12. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14. * Lesser General Public License for more details.
  15. *
  16. * You should have received a copy of the GNU Lesser General Public License
  17. * along with this program; if not, write to the Free Software Foundation,
  18. * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  19. */
  20. package org.sonar.server.measure.index;
  21. import com.google.common.collect.ArrayListMultimap;
  22. import com.google.common.collect.ImmutableList;
  23. import com.google.common.collect.ImmutableMap;
  24. import com.google.common.collect.Multimap;
  25. import java.util.HashMap;
  26. import java.util.List;
  27. import java.util.Map;
  28. import java.util.Set;
  29. import java.util.function.Function;
  30. import java.util.stream.IntStream;
  31. import java.util.stream.Stream;
  32. import javax.annotation.Nullable;
  33. import org.apache.lucene.search.join.ScoreMode;
  34. import org.elasticsearch.action.search.SearchRequestBuilder;
  35. import org.elasticsearch.action.search.SearchResponse;
  36. import org.elasticsearch.index.query.BoolQueryBuilder;
  37. import org.elasticsearch.index.query.QueryBuilder;
  38. import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
  39. import org.elasticsearch.search.aggregations.AggregationBuilders;
  40. import org.elasticsearch.search.aggregations.BucketOrder;
  41. import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket;
  42. import org.elasticsearch.search.aggregations.bucket.filter.Filter;
  43. import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregator.KeyedFilter;
  44. import org.elasticsearch.search.aggregations.bucket.nested.Nested;
  45. import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
  46. import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
  47. import org.elasticsearch.search.aggregations.bucket.terms.Terms;
  48. import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
  49. import org.elasticsearch.search.aggregations.metrics.sum.Sum;
  50. import org.elasticsearch.search.sort.FieldSortBuilder;
  51. import org.elasticsearch.search.sort.NestedSortBuilder;
  52. import org.sonar.api.server.ServerSide;
  53. import org.sonar.api.utils.System2;
  54. import org.sonar.core.util.stream.MoreCollectors;
  55. import org.sonar.server.es.EsClient;
  56. import org.sonar.server.es.SearchIdResult;
  57. import org.sonar.server.es.SearchOptions;
  58. import org.sonar.server.es.StickyFacetBuilder;
  59. import org.sonar.server.es.newindex.DefaultIndexSettingsElement;
  60. import org.sonar.server.measure.index.ProjectMeasuresQuery.MetricCriterion;
  61. import org.sonar.server.permission.index.WebAuthorizationTypeSupport;
  62. import static com.google.common.base.Preconditions.checkArgument;
  63. import static java.util.Collections.emptyList;
  64. import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
  65. import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
  66. import static org.elasticsearch.index.query.QueryBuilders.nestedQuery;
  67. import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
  68. import static org.elasticsearch.index.query.QueryBuilders.termQuery;
  69. import static org.elasticsearch.index.query.QueryBuilders.termsQuery;
  70. import static org.elasticsearch.search.aggregations.AggregationBuilders.filters;
  71. import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
  72. import static org.elasticsearch.search.sort.SortOrder.ASC;
  73. import static org.elasticsearch.search.sort.SortOrder.DESC;
  74. import static org.sonar.api.measures.CoreMetrics.ALERT_STATUS_KEY;
  75. import static org.sonar.api.measures.CoreMetrics.COVERAGE_KEY;
  76. import static org.sonar.api.measures.CoreMetrics.DUPLICATED_LINES_DENSITY_KEY;
  77. import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY;
  78. import static org.sonar.api.measures.CoreMetrics.NEW_COVERAGE_KEY;
  79. import static org.sonar.api.measures.CoreMetrics.NEW_DUPLICATED_LINES_DENSITY_KEY;
  80. import static org.sonar.api.measures.CoreMetrics.NEW_LINES_KEY;
  81. import static org.sonar.api.measures.CoreMetrics.NEW_MAINTAINABILITY_RATING_KEY;
  82. import static org.sonar.api.measures.CoreMetrics.NEW_RELIABILITY_RATING_KEY;
  83. import static org.sonar.api.measures.CoreMetrics.NEW_SECURITY_RATING_KEY;
  84. import static org.sonar.api.measures.CoreMetrics.RELIABILITY_RATING_KEY;
  85. import static org.sonar.api.measures.CoreMetrics.SECURITY_RATING_KEY;
  86. import static org.sonar.api.measures.CoreMetrics.SQALE_RATING_KEY;
  87. import static org.sonar.server.es.EsUtils.escapeSpecialRegexChars;
  88. import static org.sonar.server.es.EsUtils.termsToMap;
  89. import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE;
  90. import static org.sonar.server.measure.index.ProjectMeasuresDoc.QUALITY_GATE_STATUS;
  91. import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_ANALYSED_AT;
  92. import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_KEY;
  93. import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_LANGUAGES;
  94. import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_MEASURES;
  95. import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_NAME;
  96. import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_NCLOC_LANGUAGE_DISTRIBUTION;
  97. import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_ORGANIZATION_UUID;
  98. import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_QUALITY_GATE_STATUS;
  99. import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_TAGS;
  100. import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES;
  101. import static org.sonar.server.measure.index.ProjectMeasuresQuery.SORT_BY_LAST_ANALYSIS_DATE;
  102. import static org.sonar.server.measure.index.ProjectMeasuresQuery.SORT_BY_NAME;
  103. import static org.sonarqube.ws.client.project.ProjectsWsParameters.FILTER_LANGUAGES;
  104. import static org.sonarqube.ws.client.project.ProjectsWsParameters.FILTER_TAGS;
  105. import static org.sonarqube.ws.client.project.ProjectsWsParameters.MAX_PAGE_SIZE;
  106. @ServerSide
  107. public class ProjectMeasuresIndex {
  108. public static final List<String> SUPPORTED_FACETS = ImmutableList.of(
  109. NCLOC_KEY,
  110. NEW_LINES_KEY,
  111. DUPLICATED_LINES_DENSITY_KEY,
  112. NEW_DUPLICATED_LINES_DENSITY_KEY,
  113. COVERAGE_KEY,
  114. NEW_COVERAGE_KEY,
  115. SQALE_RATING_KEY,
  116. NEW_MAINTAINABILITY_RATING_KEY,
  117. RELIABILITY_RATING_KEY,
  118. NEW_RELIABILITY_RATING_KEY,
  119. SECURITY_RATING_KEY,
  120. NEW_SECURITY_RATING_KEY,
  121. ALERT_STATUS_KEY,
  122. FILTER_LANGUAGES,
  123. FILTER_TAGS);
  124. private static final Double[] LINES_THRESHOLDS = new Double[] {1_000d, 10_000d, 100_000d, 500_000d};
  125. private static final Double[] COVERAGE_THRESHOLDS = new Double[] {30d, 50d, 70d, 80d};
  126. private static final Double[] DUPLICATIONS_THRESHOLDS = new Double[] {3d, 5d, 10d, 20d};
  127. private static final String FIELD_MEASURES_KEY = FIELD_MEASURES + "." + ProjectMeasuresIndexDefinition.FIELD_MEASURES_KEY;
  128. private static final String FIELD_MEASURES_VALUE = FIELD_MEASURES + "." + ProjectMeasuresIndexDefinition.FIELD_MEASURES_VALUE;
  129. private static final String FIELD_DISTRIB_LANGUAGE = FIELD_NCLOC_LANGUAGE_DISTRIBUTION + "." + ProjectMeasuresIndexDefinition.FIELD_DISTRIB_LANGUAGE;
  130. private static final String FIELD_DISTRIB_NCLOC = FIELD_NCLOC_LANGUAGE_DISTRIBUTION + "." + ProjectMeasuresIndexDefinition.FIELD_DISTRIB_NCLOC;
  131. private static final Map<String, FacetSetter> FACET_FACTORIES = ImmutableMap.<String, FacetSetter>builder()
  132. .put(NCLOC_KEY, (esSearch, query, facetBuilder) -> addRangeFacet(esSearch, NCLOC_KEY, facetBuilder, LINES_THRESHOLDS))
  133. .put(NEW_LINES_KEY, (esSearch, query, facetBuilder) -> addRangeFacet(esSearch, NEW_LINES_KEY, facetBuilder, LINES_THRESHOLDS))
  134. .put(DUPLICATED_LINES_DENSITY_KEY,
  135. (esSearch, query, facetBuilder) -> addRangeFacetIncludingNoData(esSearch, DUPLICATED_LINES_DENSITY_KEY, facetBuilder, DUPLICATIONS_THRESHOLDS))
  136. .put(NEW_DUPLICATED_LINES_DENSITY_KEY,
  137. (esSearch, query, facetBuilder) -> addRangeFacetIncludingNoData(esSearch, NEW_DUPLICATED_LINES_DENSITY_KEY, facetBuilder, DUPLICATIONS_THRESHOLDS))
  138. .put(COVERAGE_KEY, (esSearch, query, facetBuilder) -> addRangeFacetIncludingNoData(esSearch, COVERAGE_KEY, facetBuilder, COVERAGE_THRESHOLDS))
  139. .put(NEW_COVERAGE_KEY, (esSearch, query, facetBuilder) -> addRangeFacetIncludingNoData(esSearch, NEW_COVERAGE_KEY, facetBuilder, COVERAGE_THRESHOLDS))
  140. .put(SQALE_RATING_KEY, (esSearch, query, facetBuilder) -> addRatingFacet(esSearch, SQALE_RATING_KEY, facetBuilder))
  141. .put(NEW_MAINTAINABILITY_RATING_KEY, (esSearch, query, facetBuilder) -> addRatingFacet(esSearch, NEW_MAINTAINABILITY_RATING_KEY, facetBuilder))
  142. .put(RELIABILITY_RATING_KEY, (esSearch, query, facetBuilder) -> addRatingFacet(esSearch, RELIABILITY_RATING_KEY, facetBuilder))
  143. .put(NEW_RELIABILITY_RATING_KEY, (esSearch, query, facetBuilder) -> addRatingFacet(esSearch, NEW_RELIABILITY_RATING_KEY, facetBuilder))
  144. .put(SECURITY_RATING_KEY, (esSearch, query, facetBuilder) -> addRatingFacet(esSearch, SECURITY_RATING_KEY, facetBuilder))
  145. .put(NEW_SECURITY_RATING_KEY, (esSearch, query, facetBuilder) -> addRatingFacet(esSearch, NEW_SECURITY_RATING_KEY, facetBuilder))
  146. .put(ALERT_STATUS_KEY, (esSearch, query, facetBuilder) -> esSearch.addAggregation(createStickyFacet(ALERT_STATUS_KEY, facetBuilder, createQualityGateFacet())))
  147. .put(FILTER_LANGUAGES, ProjectMeasuresIndex::addLanguagesFacet)
  148. .put(FIELD_TAGS, ProjectMeasuresIndex::addTagsFacet)
  149. .build();
  150. private final EsClient client;
  151. private final WebAuthorizationTypeSupport authorizationTypeSupport;
  152. private final System2 system2;
  153. public ProjectMeasuresIndex(EsClient client, WebAuthorizationTypeSupport authorizationTypeSupport, System2 system2) {
  154. this.client = client;
  155. this.authorizationTypeSupport = authorizationTypeSupport;
  156. this.system2 = system2;
  157. }
  158. public SearchIdResult<String> search(ProjectMeasuresQuery query, SearchOptions searchOptions) {
  159. SearchRequestBuilder requestBuilder = client
  160. .prepareSearch(TYPE_PROJECT_MEASURES.getMainType())
  161. .setFetchSource(false)
  162. .setFrom(searchOptions.getOffset())
  163. .setSize(searchOptions.getLimit());
  164. BoolQueryBuilder esFilter = boolQuery();
  165. Map<String, QueryBuilder> filters = createFilters(query);
  166. filters.values().forEach(esFilter::must);
  167. requestBuilder.setQuery(esFilter);
  168. addFacets(requestBuilder, searchOptions, filters, query);
  169. addSort(query, requestBuilder);
  170. return new SearchIdResult<>(requestBuilder.get(), id -> id, system2.getDefaultTimeZone());
  171. }
  172. public ProjectMeasuresStatistics searchTelemetryStatistics() {
  173. SearchRequestBuilder request = client
  174. .prepareSearch(TYPE_PROJECT_MEASURES.getMainType())
  175. .setFetchSource(false)
  176. .setSize(0);
  177. BoolQueryBuilder esFilter = boolQuery();
  178. request.setQuery(esFilter);
  179. request.addAggregation(AggregationBuilders.terms(FIELD_LANGUAGES)
  180. .field(FIELD_LANGUAGES)
  181. .size(MAX_PAGE_SIZE)
  182. .minDocCount(1)
  183. .order(BucketOrder.count(false)));
  184. request.addAggregation(AggregationBuilders.nested(FIELD_NCLOC_LANGUAGE_DISTRIBUTION, FIELD_NCLOC_LANGUAGE_DISTRIBUTION)
  185. .subAggregation(AggregationBuilders.terms(FIELD_NCLOC_LANGUAGE_DISTRIBUTION + "_terms")
  186. .field(FIELD_DISTRIB_LANGUAGE)
  187. .size(MAX_PAGE_SIZE)
  188. .minDocCount(1)
  189. .order(BucketOrder.count(false))
  190. .subAggregation(sum(FIELD_DISTRIB_NCLOC).field(FIELD_DISTRIB_NCLOC))));
  191. request.addAggregation(AggregationBuilders.nested(NCLOC_KEY, FIELD_MEASURES)
  192. .subAggregation(AggregationBuilders.filter(NCLOC_KEY + "_filter", termQuery(FIELD_MEASURES_KEY, NCLOC_KEY))
  193. .subAggregation(sum(NCLOC_KEY + "_filter_sum").field(FIELD_MEASURES_VALUE))));
  194. ProjectMeasuresStatistics.Builder statistics = ProjectMeasuresStatistics.builder();
  195. SearchResponse response = request.get();
  196. statistics.setProjectCount(response.getHits().getTotalHits());
  197. Stream.of(NCLOC_KEY)
  198. .map(metric -> (Nested) response.getAggregations().get(metric))
  199. .map(nested -> (Filter) nested.getAggregations().get(nested.getName() + "_filter"))
  200. .map(filter -> (Sum) filter.getAggregations().get(filter.getName() + "_sum"))
  201. .forEach(sum -> {
  202. String metric = sum.getName().replace("_filter_sum", "");
  203. long value = Math.round(sum.getValue());
  204. statistics.setSum(metric, value);
  205. });
  206. statistics.setProjectCountByLanguage(termsToMap(response.getAggregations().get(FIELD_LANGUAGES)));
  207. Function<Terms.Bucket, Long> bucketToNcloc = bucket -> Math.round(((Sum) bucket.getAggregations().get(FIELD_DISTRIB_NCLOC)).getValue());
  208. Map<String, Long> nclocByLanguage = Stream.of((Nested) response.getAggregations().get(FIELD_NCLOC_LANGUAGE_DISTRIBUTION))
  209. .map(nested -> (Terms) nested.getAggregations().get(nested.getName() + "_terms"))
  210. .flatMap(terms -> terms.getBuckets().stream())
  211. .collect(MoreCollectors.uniqueIndex(Bucket::getKeyAsString, bucketToNcloc));
  212. statistics.setNclocByLanguage(nclocByLanguage);
  213. return statistics.build();
  214. }
  215. private static void addSort(ProjectMeasuresQuery query, SearchRequestBuilder requestBuilder) {
  216. String sort = query.getSort();
  217. if (SORT_BY_NAME.equals(sort)) {
  218. requestBuilder.addSort(DefaultIndexSettingsElement.SORTABLE_ANALYZER.subField(FIELD_NAME), query.isAsc() ? ASC : DESC);
  219. } else if (SORT_BY_LAST_ANALYSIS_DATE.equals(sort)) {
  220. requestBuilder.addSort(FIELD_ANALYSED_AT, query.isAsc() ? ASC : DESC);
  221. } else if (ALERT_STATUS_KEY.equals(sort)) {
  222. requestBuilder.addSort(FIELD_QUALITY_GATE_STATUS, query.isAsc() ? ASC : DESC);
  223. requestBuilder.addSort(DefaultIndexSettingsElement.SORTABLE_ANALYZER.subField(FIELD_NAME), ASC);
  224. } else {
  225. addMetricSort(query, requestBuilder, sort);
  226. requestBuilder.addSort(DefaultIndexSettingsElement.SORTABLE_ANALYZER.subField(FIELD_NAME), ASC);
  227. }
  228. // last sort is by key in order to be deterministic when same value
  229. requestBuilder.addSort(FIELD_KEY, ASC);
  230. }
  231. private static void addMetricSort(ProjectMeasuresQuery query, SearchRequestBuilder requestBuilder, String sort) {
  232. requestBuilder.addSort(
  233. new FieldSortBuilder(FIELD_MEASURES_VALUE)
  234. .setNestedSort(
  235. new NestedSortBuilder(FIELD_MEASURES)
  236. .setFilter(termQuery(FIELD_MEASURES_KEY, sort)))
  237. .order(query.isAsc() ? ASC : DESC));
  238. }
  239. private static void addRangeFacet(SearchRequestBuilder esSearch, String metricKey, StickyFacetBuilder facetBuilder, Double... thresholds) {
  240. esSearch.addAggregation(createStickyFacet(metricKey, facetBuilder, createRangeFacet(metricKey, thresholds)));
  241. }
  242. private static void addRangeFacetIncludingNoData(SearchRequestBuilder esSearch, String metricKey, StickyFacetBuilder facetBuilder, Double... thresholds) {
  243. esSearch.addAggregation(createStickyFacet(metricKey, facetBuilder,
  244. AggregationBuilders.filter("combined_" + metricKey, matchAllQuery())
  245. .subAggregation(createRangeFacet(metricKey, thresholds))
  246. .subAggregation(createNoDataFacet(metricKey))));
  247. }
  248. private static void addRatingFacet(SearchRequestBuilder esSearch, String metricKey, StickyFacetBuilder facetBuilder) {
  249. esSearch.addAggregation(createStickyFacet(metricKey, facetBuilder, createRatingFacet(metricKey)));
  250. }
  251. private static void addLanguagesFacet(SearchRequestBuilder esSearch, ProjectMeasuresQuery query, StickyFacetBuilder facetBuilder) {
  252. esSearch.addAggregation(facetBuilder.buildStickyFacet(FIELD_LANGUAGES, FILTER_LANGUAGES, query.getLanguages().map(Set::toArray).orElseGet(() -> new Object[] {})));
  253. }
  254. private static void addTagsFacet(SearchRequestBuilder esSearch, ProjectMeasuresQuery query, StickyFacetBuilder facetBuilder) {
  255. esSearch.addAggregation(facetBuilder.buildStickyFacet(FIELD_TAGS, FILTER_TAGS, query.getTags().map(Set::toArray).orElseGet(() -> new Object[] {})));
  256. }
  257. private static void addFacets(SearchRequestBuilder esSearch, SearchOptions options, Map<String, QueryBuilder> filters, ProjectMeasuresQuery query) {
  258. StickyFacetBuilder facetBuilder = new StickyFacetBuilder(matchAllQuery(), filters);
  259. options.getFacets().stream()
  260. .filter(FACET_FACTORIES::containsKey)
  261. .map(FACET_FACTORIES::get)
  262. .forEach(factory -> factory.addFacet(esSearch, query, facetBuilder));
  263. }
  264. private static AbstractAggregationBuilder createStickyFacet(String facetKey, StickyFacetBuilder facetBuilder, AbstractAggregationBuilder aggregationBuilder) {
  265. BoolQueryBuilder facetFilter = facetBuilder.getStickyFacetFilter(facetKey);
  266. return AggregationBuilders
  267. .global(facetKey)
  268. .subAggregation(
  269. AggregationBuilders
  270. .filter("facet_filter_" + facetKey, facetFilter)
  271. .subAggregation(aggregationBuilder));
  272. }
  273. private static AbstractAggregationBuilder createRangeFacet(String metricKey, Double... thresholds) {
  274. RangeAggregationBuilder rangeAgg = AggregationBuilders.range(metricKey)
  275. .field(FIELD_MEASURES_VALUE);
  276. final int lastIndex = thresholds.length - 1;
  277. IntStream.range(0, thresholds.length)
  278. .forEach(i -> {
  279. if (i == 0) {
  280. rangeAgg.addUnboundedTo(thresholds[0]);
  281. rangeAgg.addRange(thresholds[0], thresholds[1]);
  282. } else if (i == lastIndex) {
  283. rangeAgg.addUnboundedFrom(thresholds[lastIndex]);
  284. } else {
  285. rangeAgg.addRange(thresholds[i], thresholds[i + 1]);
  286. }
  287. });
  288. return AggregationBuilders.nested("nested_" + metricKey, FIELD_MEASURES)
  289. .subAggregation(
  290. AggregationBuilders.filter("filter_" + metricKey, termsQuery(FIELD_MEASURES_KEY, metricKey))
  291. .subAggregation(rangeAgg));
  292. }
  293. private static AbstractAggregationBuilder createNoDataFacet(String metricKey) {
  294. return AggregationBuilders.filter(
  295. "no_data_" + metricKey,
  296. boolQuery().mustNot(nestedQuery(FIELD_MEASURES, termQuery(FIELD_MEASURES_KEY, metricKey), ScoreMode.Avg)));
  297. }
  298. private static AbstractAggregationBuilder createRatingFacet(String metricKey) {
  299. return AggregationBuilders.nested("nested_" + metricKey, FIELD_MEASURES)
  300. .subAggregation(
  301. AggregationBuilders.filter("filter_" + metricKey, termsQuery(FIELD_MEASURES_KEY, metricKey))
  302. .subAggregation(filters(metricKey,
  303. new KeyedFilter("1", termQuery(FIELD_MEASURES_VALUE, 1d)),
  304. new KeyedFilter("2", termQuery(FIELD_MEASURES_VALUE, 2d)),
  305. new KeyedFilter("3", termQuery(FIELD_MEASURES_VALUE, 3d)),
  306. new KeyedFilter("4", termQuery(FIELD_MEASURES_VALUE, 4d)),
  307. new KeyedFilter("5", termQuery(FIELD_MEASURES_VALUE, 5d)))));
  308. }
  309. private static AbstractAggregationBuilder createQualityGateFacet() {
  310. return filters(
  311. ALERT_STATUS_KEY,
  312. QUALITY_GATE_STATUS.entrySet().stream()
  313. .map(entry -> new KeyedFilter(entry.getKey(), termQuery(FIELD_QUALITY_GATE_STATUS, entry.getValue())))
  314. .toArray(KeyedFilter[]::new));
  315. }
  316. private Map<String, QueryBuilder> createFilters(ProjectMeasuresQuery query) {
  317. Map<String, QueryBuilder> filters = new HashMap<>();
  318. filters.put("__indexType", termQuery(FIELD_INDEX_TYPE, TYPE_PROJECT_MEASURES.getName()));
  319. if (!query.isIgnoreAuthorization()) {
  320. filters.put("__authorization", authorizationTypeSupport.createQueryFilter());
  321. }
  322. Multimap<String, MetricCriterion> metricCriterionMultimap = ArrayListMultimap.create();
  323. query.getMetricCriteria().forEach(metricCriterion -> metricCriterionMultimap.put(metricCriterion.getMetricKey(), metricCriterion));
  324. metricCriterionMultimap.asMap().forEach((key, value) -> {
  325. BoolQueryBuilder metricFilters = boolQuery();
  326. value
  327. .stream()
  328. .map(ProjectMeasuresIndex::toQuery)
  329. .forEach(metricFilters::must);
  330. filters.put(key, metricFilters);
  331. });
  332. query.getQualityGateStatus()
  333. .ifPresent(qualityGateStatus -> filters.put(ALERT_STATUS_KEY, termQuery(FIELD_QUALITY_GATE_STATUS, QUALITY_GATE_STATUS.get(qualityGateStatus.name()))));
  334. query.getProjectUuids()
  335. .ifPresent(projectUuids -> filters.put("ids", termsQuery("_id", projectUuids)));
  336. query.getLanguages()
  337. .ifPresent(languages -> filters.put(FILTER_LANGUAGES, termsQuery(FIELD_LANGUAGES, languages)));
  338. query.getOrganizationUuid()
  339. .ifPresent(organizationUuid -> filters.put(FIELD_ORGANIZATION_UUID, termQuery(FIELD_ORGANIZATION_UUID, organizationUuid)));
  340. query.getTags()
  341. .ifPresent(tags -> filters.put(FIELD_TAGS, termsQuery(FIELD_TAGS, tags)));
  342. query.getQueryText()
  343. .map(ProjectsTextSearchQueryFactory::createQuery)
  344. .ifPresent(queryBuilder -> filters.put("textQuery", queryBuilder));
  345. return filters;
  346. }
  347. private static QueryBuilder toQuery(MetricCriterion criterion) {
  348. if (criterion.isNoData()) {
  349. return boolQuery().mustNot(
  350. nestedQuery(
  351. FIELD_MEASURES,
  352. termQuery(FIELD_MEASURES_KEY, criterion.getMetricKey()),
  353. ScoreMode.Avg));
  354. }
  355. return nestedQuery(
  356. FIELD_MEASURES,
  357. boolQuery()
  358. .filter(termQuery(FIELD_MEASURES_KEY, criterion.getMetricKey()))
  359. .filter(toValueQuery(criterion)),
  360. ScoreMode.Avg);
  361. }
  362. private static QueryBuilder toValueQuery(MetricCriterion criterion) {
  363. String fieldName = FIELD_MEASURES_VALUE;
  364. switch (criterion.getOperator()) {
  365. case GT:
  366. return rangeQuery(fieldName).gt(criterion.getValue());
  367. case GTE:
  368. return rangeQuery(fieldName).gte(criterion.getValue());
  369. case LT:
  370. return rangeQuery(fieldName).lt(criterion.getValue());
  371. case LTE:
  372. return rangeQuery(fieldName).lte(criterion.getValue());
  373. case EQ:
  374. return termQuery(fieldName, criterion.getValue());
  375. default:
  376. throw new IllegalStateException("Metric criteria non supported: " + criterion.getOperator().name());
  377. }
  378. }
  379. public List<String> searchTags(@Nullable String textQuery, int size) {
  380. int maxPageSize = 500;
  381. checkArgument(size <= maxPageSize, "Page size must be lower than or equals to " + maxPageSize);
  382. if (size <= 0) {
  383. return emptyList();
  384. }
  385. TermsAggregationBuilder tagFacet = AggregationBuilders.terms(FIELD_TAGS)
  386. .field(FIELD_TAGS)
  387. .size(size)
  388. .minDocCount(1)
  389. .order(BucketOrder.key(true));
  390. if (textQuery != null) {
  391. tagFacet.includeExclude(new IncludeExclude(".*" + escapeSpecialRegexChars(textQuery) + ".*", null));
  392. }
  393. SearchRequestBuilder searchQuery = client
  394. .prepareSearch(TYPE_PROJECT_MEASURES.getMainType())
  395. .setQuery(authorizationTypeSupport.createQueryFilter())
  396. .setFetchSource(false)
  397. .setSize(0)
  398. .addAggregation(tagFacet);
  399. Terms aggregation = searchQuery.get().getAggregations().get(FIELD_TAGS);
  400. return aggregation.getBuckets().stream()
  401. .map(Bucket::getKeyAsString)
  402. .collect(MoreCollectors.toList());
  403. }
  404. @FunctionalInterface
  405. private interface FacetSetter {
  406. void addFacet(SearchRequestBuilder esSearch, ProjectMeasuresQuery query, StickyFacetBuilder facetBuilder);
  407. }
  408. }