From d4c3bbb1e8303d87260fc046841a84483711b7d3 Mon Sep 17 00:00:00 2001 From: lukasz-jarocki-sonarsource Date: Mon, 21 Aug 2023 16:37:34 +0200 Subject: [PATCH] SONAR-20198 Added rules data to elasticsearch indexes --- .../org/sonar/db/rule/RuleForIndexingDto.java | 26 ++++ .../sonar/db/rule/RuleForIndexingDtoTest.java | 50 ++++++ .../java/org/sonar/db/rule/RuleTesting.java | 9 ++ .../rule/index/RuleIndexDefinitionIT.java | 4 +- .../sonar/server/rule/index/RuleIndexIT.java | 143 ++++++++++++++++++ .../sonar/server/es/StickyFacetBuilder.java | 9 ++ .../sonar/server/issue/index/IssueDoc.java | 4 +- .../org/sonar/server/rule/index/RuleDoc.java | 28 +++- .../sonar/server/rule/index/RuleIndex.java | 122 +++++++++++++-- .../rule/index/RuleIndexDefinition.java | 16 +- .../sonar/server/rule/index/RuleQuery.java | 32 ++++ .../sonar/server/issue/index/IssueIndex.java | 6 +- .../server/rule/ws/RulesWsParameters.java | 4 + 13 files changed, 434 insertions(+), 19 deletions(-) create mode 100644 server/sonar-db-dao/src/test/java/org/sonar/db/rule/RuleForIndexingDtoTest.java diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleForIndexingDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleForIndexingDto.java index 8f939d37bd0..1b6e730b315 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleForIndexingDto.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleForIndexingDto.java @@ -28,6 +28,7 @@ import javax.annotation.CheckForNull; import org.sonar.api.rule.RuleKey; import org.sonar.api.rule.RuleStatus; import org.sonar.api.rules.RuleType; +import org.sonar.db.issue.ImpactDto; public class RuleForIndexingDto { @@ -54,6 +55,9 @@ public class RuleForIndexingDto { private long updatedAt; private Set ruleDescriptionSectionsDtos = new HashSet<>(); + private String cleanCodeAttributeCategory; + private Set impacts = new HashSet<>(); + @VisibleForTesting public RuleForIndexingDto() { // nothing to do here @@ -82,6 +86,12 @@ public class RuleForIndexingDto { if (r.getRuleDescriptionSectionDtos() != null) { ruleForIndexingDto.setRuleDescriptionSectionsDtos(Sets.newHashSet(r.getRuleDescriptionSectionDtos())); } + + if (r.getCleanCodeAttribute() != null) { + ruleForIndexingDto.cleanCodeAttributeCategory = r.getCleanCodeAttribute().getAttributeCategory().name(); + } + ruleForIndexingDto.setImpacts(r.getDefaultImpacts()); + return ruleForIndexingDto; } @@ -205,4 +215,20 @@ public class RuleForIndexingDto { public void setType(int type) { this.type = type; } + + public String getCleanCodeAttributeCategory() { + return cleanCodeAttributeCategory; + } + + public void setCleanCodeAttributeCategory(String cleanCodeAttributeCategory) { + this.cleanCodeAttributeCategory = cleanCodeAttributeCategory; + } + + public Set getImpacts() { + return impacts; + } + + public void setImpacts(Set impacts) { + this.impacts = impacts; + } } diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/rule/RuleForIndexingDtoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/rule/RuleForIndexingDtoTest.java new file mode 100644 index 00000000000..995b26d195b --- /dev/null +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/rule/RuleForIndexingDtoTest.java @@ -0,0 +1,50 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.rule; + +import java.util.List; +import org.junit.Test; +import org.sonar.api.issue.impact.Severity; +import org.sonar.api.issue.impact.SoftwareQuality; +import org.sonar.api.rules.CleanCodeAttribute; +import org.sonar.api.rules.CleanCodeAttributeCategory; +import org.sonar.db.issue.ImpactDto; + +import static org.assertj.core.api.Assertions.assertThat; + +public class RuleForIndexingDtoTest { + + @Test + public void fromRuleDto_whenCleanCodeAttributeSet_setCleanCodeCategory() { + RuleDto ruleDto = RuleTesting.newRuleWithoutDescriptionSection(); + ruleDto.setCleanCodeAttribute(CleanCodeAttribute.FOCUSED); + ImpactDto impactDto = new ImpactDto().setSeverity(Severity.HIGH).setSoftwareQuality(SoftwareQuality.SECURITY); + ruleDto.replaceAllDefaultImpacts(List.of(impactDto)); + + RuleForIndexingDto ruleForIndexingDto = RuleForIndexingDto.fromRuleDto(ruleDto); + + assertThat(ruleForIndexingDto.getCleanCodeAttributeCategory()).isEqualTo(CleanCodeAttributeCategory.ADAPTABLE.name()); + ImpactDto impact = ruleForIndexingDto.getImpacts().iterator().next(); + + assertThat(impact.getSeverity()).isEqualTo(Severity.HIGH); + assertThat(impact.getSoftwareQuality()).isEqualTo(SoftwareQuality.SECURITY); + } + +} \ No newline at end of file diff --git a/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/rule/RuleTesting.java b/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/rule/RuleTesting.java index 4e376373f5a..ac58146fc04 100644 --- a/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/rule/RuleTesting.java +++ b/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/rule/RuleTesting.java @@ -19,6 +19,7 @@ */ package org.sonar.db.rule; +import java.util.Collection; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; @@ -242,4 +243,12 @@ public class RuleTesting { return rule -> rule.setTags(copyOf(tags)); } + public static Consumer setCleanCodeAttribute(CleanCodeAttribute cleanCodeAttribute) { + return rule -> rule.setCleanCodeAttribute(cleanCodeAttribute); + } + + public static Consumer setImpacts(Collection impacts) { + return rule -> rule.replaceAllDefaultImpacts(impacts); + } + } diff --git a/server/sonar-server-common/src/it/java/org/sonar/server/rule/index/RuleIndexDefinitionIT.java b/server/sonar-server-common/src/it/java/org/sonar/server/rule/index/RuleIndexDefinitionIT.java index b90fe9a2f77..0a9d03fe16f 100644 --- a/server/sonar-server-common/src/it/java/org/sonar/server/rule/index/RuleIndexDefinitionIT.java +++ b/server/sonar-server-common/src/it/java/org/sonar/server/rule/index/RuleIndexDefinitionIT.java @@ -19,9 +19,9 @@ */ package org.sonar.server.rule.index; -import com.google.common.collect.ImmutableMap; import java.io.IOException; import java.util.List; +import java.util.Map; import org.apache.commons.lang.StringUtils; import org.apache.lucene.search.TotalHits; import org.elasticsearch.client.RequestOptions; @@ -93,7 +93,7 @@ public class RuleIndexDefinitionIT { "quick", "brown", "fox", "jump", "over", "lazi", "dog"); // the following method fails if PUT fails - tester.putDocuments(TYPE_RULE, new RuleDoc(ImmutableMap.of( + tester.putDocuments(TYPE_RULE, new RuleDoc(Map.of( FIELD_RULE_UUID, "123", FIELD_RULE_HTML_DESCRIPTION, longText, FIELD_RULE_REPOSITORY, "java", diff --git a/server/sonar-server-common/src/it/java/org/sonar/server/rule/index/RuleIndexIT.java b/server/sonar-server-common/src/it/java/org/sonar/server/rule/index/RuleIndexIT.java index 6a391e43abb..5292405158e 100644 --- a/server/sonar-server-common/src/it/java/org/sonar/server/rule/index/RuleIndexIT.java +++ b/server/sonar-server-common/src/it/java/org/sonar/server/rule/index/RuleIndexIT.java @@ -30,12 +30,16 @@ import java.util.function.Consumer; import org.junit.Rule; import org.junit.Test; import org.sonar.api.impl.utils.AlwaysIncreasingSystem2; +import org.sonar.api.issue.impact.Severity; +import org.sonar.api.issue.impact.SoftwareQuality; import org.sonar.api.rule.RuleKey; import org.sonar.api.rule.RuleStatus; +import org.sonar.api.rules.CleanCodeAttribute; import org.sonar.api.utils.System2; import org.sonar.core.util.UuidFactory; import org.sonar.core.util.UuidFactoryFast; import org.sonar.db.DbTester; +import org.sonar.db.issue.ImpactDto; import org.sonar.db.qualityprofile.QProfileDto; import org.sonar.db.rule.RuleDto; import org.sonar.server.es.EsTester; @@ -66,7 +70,9 @@ import static org.sonar.api.rules.RuleType.SECURITY_HOTSPOT; import static org.sonar.api.rules.RuleType.VULNERABILITY; import static org.sonar.db.rule.RuleDescriptionSectionDto.createDefaultRuleDescriptionSection; import static org.sonar.db.rule.RuleTesting.newRule; +import static org.sonar.db.rule.RuleTesting.setCleanCodeAttribute; import static org.sonar.db.rule.RuleTesting.setCreatedAt; +import static org.sonar.db.rule.RuleTesting.setImpacts; import static org.sonar.db.rule.RuleTesting.setIsExternal; import static org.sonar.db.rule.RuleTesting.setIsTemplate; import static org.sonar.db.rule.RuleTesting.setLanguage; @@ -780,6 +786,143 @@ public class RuleIndexIT { verifyEmptySearch(availableSinceNowQuery); } + @Test + public void search_by_clean_code_attribute() { + RuleDto ruleDto = createRule(setRepositoryKey("php"), setCleanCodeAttribute(CleanCodeAttribute.FOCUSED)); + index(); + + RuleQuery query = new RuleQuery(); + query.setCleanCodeAttributesCategory(CleanCodeAttribute.LOGICAL.getAttributeCategory().name()); + SearchIdResult result1 = underTest.search(query, new SearchOptions()); + assertThat(result1.getUuids()).isEmpty(); + + + query = new RuleQuery(); + query.setCleanCodeAttributesCategory(CleanCodeAttribute.FOCUSED.getAttributeCategory().name()); + + SearchIdResult result2 = underTest.search(query, new SearchOptions()); + + assertThat(result2.getUuids()).containsOnly(ruleDto.getUuid()); + } + + @Test + public void search_by_software_quality() { + ImpactDto impactDto = new ImpactDto().setSoftwareQuality(SoftwareQuality.SECURITY).setSeverity(Severity.HIGH).setUuid("uuid"); + RuleDto phpRule = createRule(setRepositoryKey("php"), setImpacts(List.of(impactDto))); + index(); + + RuleQuery query = new RuleQuery(); + SearchIdResult result1 = underTest.search(query.setImpactSoftwareQualities(List.of(SoftwareQuality.MAINTAINABILITY.name())), new SearchOptions()); + assertThat(result1.getUuids()).isEmpty(); + + + query = new RuleQuery(); + SearchIdResult result2 = underTest.search(query.setImpactSoftwareQualities(List.of(SoftwareQuality.SECURITY.name())), new SearchOptions()); + assertThat(result2.getUuids()).containsOnly(phpRule.getUuid()); + } + + @Test + public void search_by_severity() { + ImpactDto impactDto = new ImpactDto().setSoftwareQuality(SoftwareQuality.SECURITY).setSeverity(Severity.HIGH).setUuid("uuid"); + RuleDto phpRule = createRule(setRepositoryKey("php"), setImpacts(List.of(impactDto))); + index(); + + RuleQuery query = new RuleQuery(); + SearchIdResult result1 = underTest.search(query.setImpactSeverities(List.of(Severity.MEDIUM.name())), new SearchOptions()); + assertThat(result1.getUuids()).isEmpty(); + + + query = new RuleQuery(); + SearchIdResult result2 = underTest.search(query.setImpactSeverities(List.of(Severity.HIGH.name())), new SearchOptions()); + assertThat(result2.getUuids()).containsOnly(phpRule.getUuid()); + } + + @Test + public void search_should_support_clean_code_attribute_category_facet() { + createRule(setRepositoryKey("php"), setCleanCodeAttribute(CleanCodeAttribute.FOCUSED)); + createRule(setRepositoryKey("php"), setCleanCodeAttribute(CleanCodeAttribute.LOGICAL)); + index(); + + RuleQuery query = new RuleQuery(); + + SearchIdResult result2 = underTest.search(query, new SearchOptions().addFacets(singletonList("cleanCodeAttributeCategories"))); + + assertThat(result2.getFacets().getAll()).hasSize(1); + assertThat(result2.getFacets().getAll().get("cleanCodeAttributeCategories")).containsOnly(entry("ADAPTABLE", 1L), entry("INTENTIONAL", 1L)); + } + + @Test + public void search_should_support_software_quality_facet() { + ImpactDto impactDto = new ImpactDto().setSoftwareQuality(SoftwareQuality.SECURITY).setSeverity(Severity.HIGH).setUuid("uuid"); + ImpactDto impactDto2 = new ImpactDto().setSoftwareQuality(SoftwareQuality.MAINTAINABILITY).setSeverity(Severity.LOW).setUuid("uuid2"); + createRule(setRepositoryKey("php"), setImpacts(List.of(impactDto))); + createRule(setRepositoryKey("php"), setImpacts(List.of(impactDto2))); + index(); + + RuleQuery query = new RuleQuery(); + + SearchIdResult result2 = underTest.search(query, new SearchOptions().addFacets(singletonList("impacts.softwareQuality"))); + + assertThat(result2.getFacets().getAll()).hasSize(1); + assertThat(result2.getFacets().getAll().get("impacts.softwareQuality")) + .containsOnly( + entry("SECURITY", 1L), + entry("MAINTAINABILITY", 1L), + entry("RELIABILITY", 0L)); + } + + @Test + public void search_should_support_software_quality_facet_with_filtering() { + ImpactDto impactDto = new ImpactDto().setSoftwareQuality(SoftwareQuality.SECURITY).setSeverity(Severity.HIGH).setUuid("uuid"); + ImpactDto impactDto2 = new ImpactDto().setSoftwareQuality(SoftwareQuality.MAINTAINABILITY).setSeverity(Severity.LOW).setUuid("uuid2"); + createRule(setRepositoryKey("php"), setImpacts(List.of(impactDto))); + createRule(setRepositoryKey("php"), setImpacts(List.of(impactDto2))); + index(); + + RuleQuery query = new RuleQuery(); + + SearchIdResult result2 = underTest.search(query.setImpactSeverities(Set.of(Severity.HIGH.name())), new SearchOptions().addFacets(singletonList("impacts.softwareQuality"))); + + assertThat(result2.getFacets().getAll()).hasSize(1); + assertThat(result2.getFacets().getAll().get("impacts.softwareQuality")) + .containsOnly( + entry("SECURITY", 1L), + entry("MAINTAINABILITY", 0L), + entry("RELIABILITY", 0L)); + } + + @Test + public void search_should_support_severity_facet() { + ImpactDto impactDto = new ImpactDto().setSoftwareQuality(SoftwareQuality.SECURITY).setSeverity(Severity.HIGH).setUuid("uuid"); + ImpactDto impactDto2 = new ImpactDto().setSoftwareQuality(SoftwareQuality.MAINTAINABILITY).setSeverity(Severity.LOW).setUuid("uuid2"); + createRule(setRepositoryKey("php"), setImpacts(List.of(impactDto))); + createRule(setRepositoryKey("php"), setImpacts(List.of(impactDto2))); + index(); + + RuleQuery query = new RuleQuery(); + + SearchIdResult result2 = underTest.search(query, new SearchOptions().addFacets(singletonList("impacts.severity"))); + + assertThat(result2.getFacets().getAll()).hasSize(1); + assertThat(result2.getFacets().getAll().get("impacts.severity")).containsOnly(entry("LOW", 1L), entry("MEDIUM", 0L), entry("HIGH", 1L)); + } + + @Test + public void search_should_support_severity_facet_with_filters() { + ImpactDto impactDto = new ImpactDto().setSoftwareQuality(SoftwareQuality.SECURITY).setSeverity(Severity.HIGH).setUuid("uuid"); + ImpactDto impactDto2 = new ImpactDto().setSoftwareQuality(SoftwareQuality.MAINTAINABILITY).setSeverity(Severity.LOW).setUuid("uuid2"); + createRule(setRepositoryKey("php"), setImpacts(List.of(impactDto))); + createRule(setRepositoryKey("php"), setImpacts(List.of(impactDto2))); + index(); + + RuleQuery query = new RuleQuery(); + + SearchIdResult result2 = underTest.search(query.setImpactSeverities(Set.of("LOW")), new SearchOptions().addFacets(singletonList("impacts.severity"))); + + assertThat(result2.getFacets().getAll()).hasSize(1); + assertThat(result2.getFacets().getAll().get("impacts.severity")).containsOnly(entry("LOW", 1L), entry("MEDIUM", 0L), entry("HIGH", 1L)); + } + @Test public void global_facet_on_repositories_and_tags() { createRule(setRepositoryKey("php"), setSystemTags("sysTag"), setTags()); diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/StickyFacetBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/StickyFacetBuilder.java index 5b2984c4b59..798de162480 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/StickyFacetBuilder.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/StickyFacetBuilder.java @@ -100,6 +100,15 @@ public class StickyFacetBuilder { .subAggregation(facetTopAggregation); } + public AggregationBuilder buildTopAggregationStickyFacet(String fieldName, String facetName, AggregationBuilder additionalAggregationFilter) { + BoolQueryBuilder facetFilter = getStickyFacetFilter(fieldName); + return AggregationBuilders + .global(facetName) + .subAggregation(AggregationBuilders + .filter(facetName + "_filter", facetFilter) + .subAggregation(additionalAggregationFilter)); + } + public BoolQueryBuilder getStickyFacetFilter(String... fieldNames) { BoolQueryBuilder facetFilter = boolQuery().must(query); for (Map.Entry filter : filters.entrySet()) { diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueDoc.java b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueDoc.java index 0b0fd257a1c..48d9c6e0f4d 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueDoc.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueDoc.java @@ -292,8 +292,8 @@ public class IssueDoc extends BaseDoc { return this; } - public IssueDoc setImpacts(Map softwareQualities) { - List> convertedMap = softwareQualities + public IssueDoc setImpacts(Map impacts) { + List> convertedMap = impacts .entrySet() .stream() .map(entry -> Map.of( diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleDoc.java b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleDoc.java index 28ae666cf67..766278e9bfd 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleDoc.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleDoc.java @@ -24,15 +24,20 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; import javax.annotation.CheckForNull; import javax.annotation.Nullable; import org.apache.commons.lang.builder.ReflectionToStringBuilder; +import org.sonar.api.issue.impact.SoftwareQuality; import org.sonar.api.rule.RuleKey; import org.sonar.api.rule.RuleStatus; +import org.sonar.api.rules.Rule; import org.sonar.api.rules.RuleType; +import org.sonar.db.issue.ImpactDto; import org.sonar.db.rule.RuleDescriptionSectionDto; import org.sonar.db.rule.RuleDto; import org.sonar.db.rule.RuleForIndexingDto; @@ -42,6 +47,8 @@ import org.sonar.server.security.SecurityStandards; import org.sonar.server.security.SecurityStandards.SQCategory; import static java.util.stream.Collectors.joining; +import static org.sonar.server.rule.index.RuleIndexDefinition.SUB_FIELD_SEVERITY; +import static org.sonar.server.rule.index.RuleIndexDefinition.SUB_FIELD_SOFTWARE_QUALITY; import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE; /** @@ -290,6 +297,23 @@ public class RuleDoc extends BaseDoc { return this; } + private RuleDoc setCleanCodeAttributeCategory(String cleanCodeAttributeCategory) { + setField(RuleIndexDefinition.FIELD_RULE_CLEAN_CODE_ATTRIBUTE_CATEGORY, cleanCodeAttributeCategory); + return this; + } + + public RuleDoc setImpacts(Map impacts) { + List> convertedMap = impacts + .entrySet() + .stream() + .map(entry -> Map.of( + SUB_FIELD_SOFTWARE_QUALITY, entry.getKey().name(), + SUB_FIELD_SEVERITY, entry.getValue().name())) + .toList(); + setField(RuleIndexDefinition.FIELD_RULE_IMPACTS, convertedMap); + return this; + } + @Override public String toString() { return ReflectionToStringBuilder.toString(this); @@ -318,7 +342,9 @@ public class RuleDoc extends BaseDoc { .setTags(Sets.union(dto.getTags(), dto.getSystemTags())) .setUpdatedAt(dto.getUpdatedAt()) .setHtmlDescription(getConcatenatedSectionsInHtml(dto)) - .setTemplateKey(getRuleKey(dto)); + .setTemplateKey(getRuleKey(dto)) + .setCleanCodeAttributeCategory(dto.getCleanCodeAttributeCategory()) + .setImpacts(dto.getImpacts().stream().collect(Collectors.toMap(ImpactDto::getSoftwareQuality, ImpactDto::getSeverity))); } @CheckForNull diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndex.java b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndex.java index a9438fcc768..7784d1607c8 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndex.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndex.java @@ -27,7 +27,9 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; +import javax.annotation.CheckForNull; import javax.annotation.Nullable; import org.apache.commons.lang.StringUtils; import org.apache.lucene.search.join.ScoreMode; @@ -44,12 +46,15 @@ import org.elasticsearch.join.query.JoinQueryBuilders; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.BucketOrder; +import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregator; +import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; +import org.sonar.api.issue.impact.SoftwareQuality; import org.sonar.api.rule.RuleStatus; import org.sonar.api.rule.Severity; import org.sonar.api.rules.RuleType; @@ -73,7 +78,10 @@ import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.matchPhraseQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; +import static org.elasticsearch.index.query.QueryBuilders.nestedQuery; import static org.elasticsearch.index.query.QueryBuilders.termsQuery; +import static org.elasticsearch.search.aggregations.AggregationBuilders.filters; +import static org.elasticsearch.search.aggregations.AggregationBuilders.reverseNested; import static org.sonar.api.rules.RuleType.SECURITY_HOTSPOT; import static org.sonar.api.rules.RuleType.VULNERABILITY; import static org.sonar.server.es.EsUtils.SCROLL_TIME_IN_MINUTES; @@ -88,6 +96,10 @@ import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_INHERITANCE; import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_PROFILE_UUID; import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_SEVERITY; +import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_CLEAN_CODE_ATTRIBUTE_CATEGORY; +import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_IMPACTS; +import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_IMPACT_SEVERITY; +import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_IMPACT_SOFTWARE_QUALITY; import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_CREATED_AT; import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_CWE; import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_HTML_DESCRIPTION; @@ -127,6 +139,7 @@ public class RuleIndex { public static final String FACET_TYPES = "types"; public static final String FACET_OLD_DEFAULT = "true"; public static final String FACET_CWE = "cwe"; + /** * @deprecated SansTop25 report is outdated, it has been completely deprecated in version 10.0 and will be removed from version 11.0 */ @@ -224,9 +237,9 @@ public class RuleIndex { BoolQueryBuilder textQuery = boolQuery(); JavaTokenizer.split(queryText) .stream().map(token -> boolQuery().should( - matchQuery( - SEARCH_GRAMS_ANALYZER.subField(FIELD_RULE_NAME), - StringUtils.left(token, DefaultIndexSettings.MAXIMUM_NGRAM_LENGTH)).boost(20F)) + matchQuery( + SEARCH_GRAMS_ANALYZER.subField(FIELD_RULE_NAME), + StringUtils.left(token, DefaultIndexSettings.MAXIMUM_NGRAM_LENGTH)).boost(20F)) .should( matchPhraseQuery( ENGLISH_HTML_ANALYZER.subField(FIELD_RULE_HTML_DESCRIPTION), @@ -244,7 +257,7 @@ public class RuleIndex { private static QueryBuilder termQuery(String field, String query, float boost) { return QueryBuilders.multiMatchQuery(query, - field, SEARCH_WORDS_ANALYZER.subField(field)) + field, SEARCH_WORDS_ANALYZER.subField(field)) .operator(Operator.AND) .boost(boost); } @@ -354,9 +367,36 @@ public class RuleIndex { } } + if (query.getCleanCodeAttributesCategory() != null) { + filters.put(FIELD_RULE_CLEAN_CODE_ATTRIBUTE_CATEGORY, createTermsFilter(FIELD_RULE_CLEAN_CODE_ATTRIBUTE_CATEGORY, + Set.of(query.getCleanCodeAttributesCategory()))); + } + + addImpactFilters(query, filters); + return filters; } + private static void addImpactFilters(RuleQuery query, Map allFilters) { + if (isNotEmpty(query.getImpactSoftwareQualities())) { + allFilters.put( + FIELD_RULE_IMPACT_SOFTWARE_QUALITY, + nestedQuery( + FIELD_RULE_IMPACTS, + termsQuery(FIELD_RULE_IMPACT_SOFTWARE_QUALITY, query.getImpactSoftwareQualities()), + ScoreMode.Avg)); + } + + if (isNotEmpty(query.getImpactSeverities())) { + allFilters.put( + FIELD_RULE_IMPACT_SEVERITY, + nestedQuery( + FIELD_RULE_IMPACTS, + termsQuery(FIELD_RULE_IMPACT_SEVERITY, query.getImpactSeverities()), + ScoreMode.Avg)); + } + } + private static void addSecurityStandardFilter(Map filters, String key, Collection values) { if (isNotEmpty(values)) { filters.put(key, @@ -424,7 +464,8 @@ public class RuleIndex { return filter; } - private static Map getFacets(RuleQuery query, SearchOptions options, QueryBuilder queryBuilder, Map filters) { + private static Map getFacets(RuleQuery query, SearchOptions options, QueryBuilder queryBuilder, + Map filters) { Map aggregations = new HashMap<>(); StickyFacetBuilder stickyFacetBuilder = stickyFacetBuilder(queryBuilder, filters); @@ -441,7 +482,8 @@ public class RuleIndex { return aggregations; } - private static void addDefaultFacets(RuleQuery query, SearchOptions options, Map aggregations, StickyFacetBuilder stickyFacetBuilder) { + private static void addDefaultFacets(RuleQuery query, SearchOptions options, Map aggregations, + StickyFacetBuilder stickyFacetBuilder) { if (options.getFacets().contains(FACET_LANGUAGES) || options.getFacets().contains(FACET_OLD_DEFAULT)) { Collection languages = query.getLanguages(); aggregations.put(FACET_LANGUAGES, @@ -466,16 +508,71 @@ public class RuleIndex { stickyFacetBuilder.buildStickyFacet(FIELD_RULE_REPOSITORY, FACET_REPOSITORIES, MAX_FACET_SIZE, (repositories == null) ? (new String[0]) : repositories.toArray())); } + if (options.getFacets().contains(FACET_CLEAN_CODE_ATTRIBUTE_CATEGORY)) { + Collection tags = query.getTags(); + aggregations.put(FACET_CLEAN_CODE_ATTRIBUTE_CATEGORY, + stickyFacetBuilder.buildStickyFacet(FIELD_RULE_CLEAN_CODE_ATTRIBUTE_CATEGORY, FACET_CLEAN_CODE_ATTRIBUTE_CATEGORY, MAX_FACET_SIZE, + (tags == null) ? (new String[0]) : tags.toArray())); + } + + addImpactSoftwareQualityFacetIfNeeded(options, query, aggregations, stickyFacetBuilder); + addImpactSeverityFacetIfNeeded(options, query, aggregations, stickyFacetBuilder); addDefaultSecurityFacets(query, options, aggregations, stickyFacetBuilder); } + private static void addImpactSoftwareQualityFacetIfNeeded(SearchOptions options, RuleQuery query, Map aggregations, + StickyFacetBuilder stickyFacetBuilder) { + if (!options.getFacets().contains(FACET_IMPACT_SOFTWARE_QUALITY)) { + return; + } + + Function mainQuery = softwareQuality -> boolQuery() + .filter(QueryBuilders.termQuery(FIELD_RULE_IMPACT_SOFTWARE_QUALITY, softwareQuality.name())); + + FiltersAggregator.KeyedFilter[] keyedFilters = Arrays.stream(SoftwareQuality.values()) + .map(softwareQuality -> new FiltersAggregator.KeyedFilter(softwareQuality.name(), + isNotEmpty(query.getImpactSeverities()) ? mainQuery.apply(softwareQuality) + .filter(termsQuery(FIELD_RULE_IMPACT_SEVERITY, query.getImpactSeverities())) : mainQuery.apply(softwareQuality))) + .toArray(FiltersAggregator.KeyedFilter[]::new); + + NestedAggregationBuilder nestedAggregationBuilder = AggregationBuilders.nested("nested_" + FIELD_RULE_IMPACT_SOFTWARE_QUALITY, FIELD_RULE_IMPACTS) + .subAggregation(filters(FIELD_RULE_IMPACT_SOFTWARE_QUALITY, keyedFilters)); + + AggregationBuilder aggregationBuilder = stickyFacetBuilder.buildTopAggregationStickyFacet(FIELD_RULE_IMPACT_SOFTWARE_QUALITY, FACET_IMPACT_SOFTWARE_QUALITY, nestedAggregationBuilder); + + aggregations.put(FACET_IMPACT_SOFTWARE_QUALITY, aggregationBuilder); + } + + private static void addImpactSeverityFacetIfNeeded(SearchOptions options, RuleQuery query, Map aggregations, StickyFacetBuilder stickyFacetBuilder) { + if (!options.getFacets().contains(FACET_IMPACT_SEVERITY)) { + return; + } + + Function mainQuery = softwareQuality -> boolQuery() + .filter(QueryBuilders.termQuery(FIELD_RULE_IMPACT_SEVERITY, softwareQuality.name())); + + FiltersAggregator.KeyedFilter[] keyedFilters = Arrays.stream(org.sonar.api.issue.impact.Severity.values()) + .map(severity -> new FiltersAggregator.KeyedFilter(severity.name(), + isNotEmpty(query.getImpactSoftwareQualities()) ? + mainQuery.apply(severity).filter(termsQuery(FIELD_RULE_IMPACT_SEVERITY, query.getImpactSoftwareQualities())) + : mainQuery.apply(severity))) + .toArray(FiltersAggregator.KeyedFilter[]::new); + + NestedAggregationBuilder nestedAggregationBuilder = AggregationBuilders.nested("nested_" + FIELD_RULE_IMPACT_SEVERITY, FIELD_RULE_IMPACTS) + .subAggregation(filters(FIELD_RULE_IMPACT_SEVERITY, keyedFilters).subAggregation(reverseNested("reverse_nested_" + FIELD_RULE_IMPACT_SEVERITY))); + + AggregationBuilder aggregationBuilder = stickyFacetBuilder.buildTopAggregationStickyFacet(FIELD_RULE_IMPACT_SEVERITY, FACET_IMPACT_SEVERITY, nestedAggregationBuilder); + + aggregations.put(FACET_IMPACT_SEVERITY, aggregationBuilder); + } + private static Function filterSecurityCategories() { return termsAggregation -> AggregationBuilders.filter( - "filter_by_rule_types_" + termsAggregation.getName(), - termsQuery(FIELD_RULE_TYPE, - VULNERABILITY.name(), - SECURITY_HOTSPOT.name())) + "filter_by_rule_types_" + termsAggregation.getName(), + termsQuery(FIELD_RULE_TYPE, + VULNERABILITY.name(), + SECURITY_HOTSPOT.name())) .subAggregation(termsAggregation); } @@ -629,6 +726,11 @@ public class RuleIndex { return EsUtils.termsKeys(esResponse.getAggregations().get(AGGREGATION_NAME_FOR_TAGS)); } + @CheckForNull + private static QueryBuilder createTermsFilter(String field, Collection values) { + return values.isEmpty() ? null : termsQuery(field, values); + } + private static boolean isNotEmpty(@Nullable Collection list) { return list != null && !list.isEmpty(); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java index d824901b263..928c656c2ff 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java @@ -19,7 +19,6 @@ */ package org.sonar.server.rule.index; -import com.google.common.collect.ImmutableSet; import java.util.Set; import javax.inject.Inject; import org.sonar.api.config.Configuration; @@ -68,7 +67,7 @@ public class RuleIndexDefinition implements IndexDefinition { public static final String FIELD_RULE_SONARSOURCE_SECURITY = "sonarsourceSecurity"; public static final String FIELD_RULE_TAGS = "tags"; - public static final Set SORT_FIELDS = ImmutableSet.of( + public static final Set SORT_FIELDS = Set.of( FIELD_RULE_NAME, FIELD_RULE_UPDATED_AT, FIELD_RULE_CREATED_AT, @@ -81,6 +80,13 @@ public class RuleIndexDefinition implements IndexDefinition { public static final String FIELD_ACTIVE_RULE_PROFILE_UUID = "activeRule_ruleProfile"; public static final String FIELD_ACTIVE_RULE_SEVERITY = "activeRule_severity"; + public static final String FIELD_RULE_CLEAN_CODE_ATTRIBUTE_CATEGORY = "cleanCodeAttributeCategory"; + public static final String FIELD_RULE_IMPACTS = "impacts"; + public static final String SUB_FIELD_SOFTWARE_QUALITY = "softwareQuality"; + public static final String SUB_FIELD_SEVERITY = "severity"; + public static final String FIELD_RULE_IMPACT_SOFTWARE_QUALITY = FIELD_RULE_IMPACTS + "." + SUB_FIELD_SOFTWARE_QUALITY; + public static final String FIELD_RULE_IMPACT_SEVERITY = FIELD_RULE_IMPACTS + "." + SUB_FIELD_SEVERITY; + private final Configuration config; private final boolean enableSource; @@ -149,6 +155,12 @@ public class RuleIndexDefinition implements IndexDefinition { ruleMapping.keywordFieldBuilder(FIELD_RULE_SANS_TOP_25).disableNorms().build(); ruleMapping.keywordFieldBuilder(FIELD_RULE_SONARSOURCE_SECURITY).disableNorms().build(); + ruleMapping.keywordFieldBuilder(FIELD_RULE_CLEAN_CODE_ATTRIBUTE_CATEGORY).disableNorms().build(); + ruleMapping.nestedFieldBuilder(FIELD_RULE_IMPACTS) + .addKeywordField(SUB_FIELD_SOFTWARE_QUALITY) + .addKeywordField(SUB_FIELD_SEVERITY) + .build(); + // Active rule index.createTypeMapping(TYPE_ACTIVE_RULE) .keywordFieldBuilder(FIELD_ACTIVE_RULE_UUID).disableNorms().build() diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleQuery.java b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleQuery.java index 3cab7dc9bd8..0e013e24a2d 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleQuery.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleQuery.java @@ -58,6 +58,10 @@ public class RuleQuery { private Collection sansTop25; private Collection cwe; private Collection sonarsourceSecurity; + private Collection impactSeverities; + private Collection impactSoftwareQualities; + private String cleanCodeAttributesCategory; + @CheckForNull public QProfileDto getQProfile() { @@ -337,4 +341,32 @@ public class RuleQuery { this.sonarsourceSecurity = sonarsourceSecurity; return this; } + + public Collection getImpactSeverities() { + return impactSeverities; + } + + public RuleQuery setImpactSeverities(Collection impactSeverities) { + this.impactSeverities = impactSeverities; + return this; + } + + public Collection getImpactSoftwareQualities() { + return impactSoftwareQualities; + } + + public RuleQuery setImpactSoftwareQualities(Collection impactSoftwareQualities) { + this.impactSoftwareQualities = impactSoftwareQualities; + return this; + } + + public String getCleanCodeAttributesCategory() { + return cleanCodeAttributesCategory; + } + + public RuleQuery setCleanCodeAttributesCategory(String cleanCodeAttributesCategory) { + this.cleanCodeAttributesCategory = cleanCodeAttributesCategory; + return this; + } + } diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java index f6336ef6eda..d9d19df1598 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java @@ -404,7 +404,8 @@ public class IssueIndex { return client.search(requestBuilder); } - private void configureTopAggregations(IssueQuery query, SearchOptions options, SearchSourceBuilder esRequest, AllFilters allFilters, RequestFiltersComputer filterComputer) { + private void configureTopAggregations(IssueQuery query, SearchOptions options, SearchSourceBuilder esRequest, AllFilters allFilters, + RequestFiltersComputer filterComputer) { TopAggregationHelper aggregationHelper = newAggregationHelper(filterComputer, query); configureTopAggregations(aggregationHelper, query, options, allFilters, esRequest); @@ -900,7 +901,8 @@ public class IssueIndex { esRequest.aggregation(aggregation); } - private static void addImpactSoftwareQualityFacetIfNeeded(SearchOptions options, IssueQuery query, TopAggregationHelper aggregationHelper, SearchSourceBuilder esRequest) { + private static void addImpactSoftwareQualityFacetIfNeeded(SearchOptions options, IssueQuery query, TopAggregationHelper aggregationHelper, + SearchSourceBuilder esRequest) { if (!options.getFacets().contains(PARAM_IMPACT_SOFTWARE_QUALITIES)) { return; } diff --git a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/rule/ws/RulesWsParameters.java b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/rule/ws/RulesWsParameters.java index 0c1ce5eaf99..c5356a236d3 100644 --- a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/rule/ws/RulesWsParameters.java +++ b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/rule/ws/RulesWsParameters.java @@ -48,6 +48,10 @@ public class RulesWsParameters { public static final String PARAM_TEMPLATE_KEY = "template_key"; public static final String PARAM_COMPARE_TO_PROFILE = "compareToProfile"; + public static final String PARAM_IMPACT_SOFTWARE_QUALITIES = "impactSoftwareQualities"; + public static final String PARAM_IMPACT_SEVERITIES = "impactSeverities"; + public static final String PARAM_CLEAN_CODE_ATTRIBUTE_CATEGORIES = "cleanCodeAttributeCategories"; + public static final String FIELD_REPO = "repo"; public static final String FIELD_NAME = "name"; public static final String FIELD_CREATED_AT = "createdAt"; -- 2.39.5