From: Zipeng WU Date: Tue, 9 Feb 2021 10:34:00 +0000 (+0100) Subject: SONAR-14442 Elasticsearch query for CWE Top 25 X-Git-Tag: 8.8.0.42792~142 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=9c39ab7523173d6cfcf14f1607b620ba9375678e;p=sonarqube.git SONAR-14442 Elasticsearch query for CWE Top 25 --- diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/security/SecurityStandards.java b/server/sonar-server-common/src/main/java/org/sonar/server/security/SecurityStandards.java index dfa8ad11a96..22f10f78d5a 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/security/SecurityStandards.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/security/SecurityStandards.java @@ -22,8 +22,10 @@ package org.sonar.server.security; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Ordering; + import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -70,18 +72,18 @@ public final class SecurityStandards { SANS_TOP_25_POROUS_DEFENSES, POROUS_CWE); // https://cwe.mitre.org/top25/archive/2019/2019_cwe_top25.html - private static final Set CWE_TOP25_2019 = new HashSet<>( - asList("119", "79", "20", "200", "125", "89", "416", "190", "352", + public static final List CWE_TOP25_2019 = + Collections.unmodifiableList(asList("119", "79", "20", "200", "125", "89", "416", "190", "352", "22", "78", "787", "287", "476", "732", "434", "611", "94", "798", "400", "772", "426", "502", "269", "295")); // https://cwe.mitre.org/top25/archive/2020/2020_cwe_top25.html - private static final Set CWE_TOP25_2020 = new HashSet<>( - asList("79", "787", "20", "125", "119", "89", "200", "416", "352", + public static final List CWE_TOP25_2020 = + Collections.unmodifiableList(asList("79", "787", "20", "125", "119", "89", "200", "416", "352", "78", "190", "22", "476", "287", "434", "732", "94", "522", "611", "798", "502", "269", "400", "306", "862")); - public static final Map> CWES_BY_CWE_TOP_25 = ImmutableMap.of( + public static final Map> CWES_BY_CWE_TOP_25 = ImmutableMap.of( "2019", CWE_TOP25_2019, "2020", CWE_TOP25_2020); diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java index 22008fa3dec..a948489caab 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java @@ -33,6 +33,7 @@ import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; import java.util.function.Consumer; +import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; import javax.annotation.CheckForNull; @@ -166,6 +167,7 @@ import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_VULN import static org.sonar.server.issue.index.IssueIndexDefinition.TYPE_ISSUE; import static org.sonar.server.security.SecurityReviewRating.computePercent; import static org.sonar.server.security.SecurityReviewRating.computeRating; +import static org.sonar.server.security.SecurityStandards.CWES_BY_CWE_TOP_25; import static org.sonar.server.security.SecurityStandards.SANS_TOP_25_INSECURE_INTERACTION; import static org.sonar.server.security.SecurityStandards.SANS_TOP_25_POROUS_DEFENSES; import static org.sonar.server.security.SecurityStandards.SANS_TOP_25_RISKY_RESOURCE; @@ -1016,19 +1018,32 @@ public class IssueIndex { .forEach(sansCategory -> request.aggregation(newSecurityReportSubAggregations( AggregationBuilders.filter(sansCategory, boolQuery().filter(termQuery(FIELD_ISSUE_SANS_TOP_25, sansCategory))), includeCwe, - Optional.ofNullable(SecurityStandards.CWES_BY_SANS_TOP_25.get(sansCategory))))); + SecurityStandards.CWES_BY_SANS_TOP_25.get(sansCategory)))); return processSecurityReportSearchResults(request, includeCwe); } - public List getCweTop25Reports(String uuid, boolean isViewOrApp) { - // TODO:: Mock data - SONAR-14447 elasticsearch query - return Arrays.asList( - new SecurityStandardCategoryStatistics("2019", 1, OptionalInt.empty(), 10, 5, 10, - SecurityStandards.CWES_BY_CWE_TOP_25.get("2019").stream().map(cwe -> new SecurityStandardCategoryStatistics(cwe, 1, OptionalInt.empty(), 1, 3, 2, null)) - .collect(toList())), - new SecurityStandardCategoryStatistics("2020", 0, OptionalInt.empty(), 9, 5, 10, - SecurityStandards.CWES_BY_CWE_TOP_25.get("2020").stream().map(cwe -> new SecurityStandardCategoryStatistics(cwe, 1, OptionalInt.empty(), 1, 3, 4, null)) - .collect(toList()))); + public List getCweTop25Reports(String projectUuid, boolean isViewOrApp) { + SearchSourceBuilder request = prepareNonClosedVulnerabilitiesAndHotspotSearch(projectUuid, isViewOrApp); + CWES_BY_CWE_TOP_25.keySet() + .forEach(cweYear -> request.aggregation( + newSecurityReportSubAggregations( + AggregationBuilders.filter(cweYear, boolQuery().filter(existsQuery(FIELD_ISSUE_CWE))), + true, + CWES_BY_CWE_TOP_25.get(cweYear)))); + List result = processSecurityReportSearchResults(request, true); + for (SecurityStandardCategoryStatistics cweReport : result) { + Set foundRules = cweReport.getChildren().stream() + .map(SecurityStandardCategoryStatistics::getCategory) + .collect(Collectors.toSet()); + CWES_BY_CWE_TOP_25.get(cweReport.getCategory()).stream() + .filter(rule -> !foundRules.contains(rule)) + .forEach(rule -> cweReport.getChildren().add(emptyCweStatistics(rule))); + } + return result; + } + + private static SecurityStandardCategoryStatistics emptyCweStatistics(String rule) { + return new SecurityStandardCategoryStatistics(rule, 0, OptionalInt.of(1), 0, 0, 1, null); } public List getSonarSourceReport(String projectUuid, boolean isViewOrApp, boolean includeCwe) { @@ -1038,7 +1053,7 @@ public class IssueIndex { newSecurityReportSubAggregations( AggregationBuilders.filter(sonarsourceCategory.getKey(), boolQuery().filter(termQuery(FIELD_ISSUE_SQ_SECURITY_CATEGORY, sonarsourceCategory.getKey()))), includeCwe, - Optional.ofNullable(SecurityStandards.CWES_BY_SQ_CATEGORY.get(sonarsourceCategory))))); + SecurityStandards.CWES_BY_SQ_CATEGORY.get(sonarsourceCategory)))); return processSecurityReportSearchResults(request, includeCwe); } @@ -1049,7 +1064,7 @@ public class IssueIndex { newSecurityReportSubAggregations( AggregationBuilders.filter(owaspCategory, boolQuery().filter(termQuery(FIELD_ISSUE_OWASP_TOP_10, owaspCategory))), includeCwe, - Optional.empty()))); + null))); return processSecurityReportSearchResults(request, includeCwe); } @@ -1095,16 +1110,16 @@ public class IssueIndex { reviewedSecurityHotspots, securityReviewRating, children); } - private static AggregationBuilder newSecurityReportSubAggregations(AggregationBuilder categoriesAggs, boolean includeCwe, Optional> cwesInCategory) { + private static AggregationBuilder newSecurityReportSubAggregations(AggregationBuilder categoriesAggs, boolean includeCwe, @Nullable Collection cwesInCategory) { AggregationBuilder aggregationBuilder = addSecurityReportIssueCountAggregations(categoriesAggs); if (includeCwe) { final TermsAggregationBuilder cwesAgg = AggregationBuilders.terms(AGG_CWES) .field(FIELD_ISSUE_CWE) // 100 should be enough to display all CWEs. If not, the UI will be broken anyway .size(MAX_FACET_SIZE); - cwesInCategory.ifPresent(set -> { - cwesAgg.includeExclude(new IncludeExclude(set.toArray(new String[0]), new String[0])); - }); + if (cwesInCategory != null) { + cwesAgg.includeExclude(new IncludeExclude(cwesInCategory.toArray(new String[0]), new String[0])); + } categoriesAggs.subAggregation(addSecurityReportIssueCountAggregations(cwesAgg)); } return aggregationBuilder; diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityReportsTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityReportsTest.java index a36fd41da89..58b4448e9a7 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityReportsTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityReportsTest.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; import java.util.OptionalInt; import java.util.stream.Collectors; + import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -294,6 +295,120 @@ public class IssueIndexSecurityReportsTest { assertThat(sansTop25Report).allMatch(category -> category.getChildren().isEmpty()); } + @Test + public void getCWETop25Report_aggregation() { + ComponentDto project = newPrivateProjectDto(); + indexIssues( + newDoc("openvul", project).setCwe(asList("119")).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_OPEN) + .setSeverity(Severity.MAJOR), + newDoc("notopenvul", project).setCwe(asList("119")).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_CLOSED) + .setResolution(Issue.RESOLUTION_FIXED) + .setSeverity(Severity.BLOCKER), + newDoc("toreviewhotspot", project).setCwe(asList("89")).setType(RuleType.SECURITY_HOTSPOT) + .setStatus(Issue.STATUS_TO_REVIEW), + newDoc("only2020", project).setCwe(asList("862")).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_REOPENED) + .setSeverity(Severity.MINOR), + newDoc("unknown", project).setCwe(asList("999")).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_REOPENED) + .setSeverity(Severity.MINOR)); + + List cweTop25Reports = underTest.getCweTop25Reports(project.uuid(), false); + + SecurityStandardCategoryStatistics cwe2019 = cweTop25Reports.get(0); + assertThat(cwe2019.getChildren()).hasSize(25); + assertThat(findRuleInCweByYear(cwe2019, "119")).isNotNull() + .extracting(SecurityStandardCategoryStatistics::getVulnerabilities, + SecurityStandardCategoryStatistics::getToReviewSecurityHotspots, + SecurityStandardCategoryStatistics::getReviewedSecurityHotspots) + .containsExactlyInAnyOrder(1L, 0L, 0L); + assertThat(findRuleInCweByYear(cwe2019, "89")).isNotNull() + .extracting(SecurityStandardCategoryStatistics::getVulnerabilities, + SecurityStandardCategoryStatistics::getToReviewSecurityHotspots, + SecurityStandardCategoryStatistics::getReviewedSecurityHotspots) + .containsExactlyInAnyOrder(0L, 1L, 0L); + assertThat(findRuleInCweByYear(cwe2019, "862")).isNull(); + assertThat(findRuleInCweByYear(cwe2019, "999")).isNull(); + + SecurityStandardCategoryStatistics cwe2020 = cweTop25Reports.get(1); + assertThat(cwe2020.getChildren()).hasSize(25); + assertThat(findRuleInCweByYear(cwe2020, "119")).isNotNull() + .extracting(SecurityStandardCategoryStatistics::getVulnerabilities, + SecurityStandardCategoryStatistics::getToReviewSecurityHotspots, + SecurityStandardCategoryStatistics::getReviewedSecurityHotspots) + .containsExactlyInAnyOrder(1L, 0L, 0L); + assertThat(findRuleInCweByYear(cwe2020, "89")).isNotNull() + .extracting(SecurityStandardCategoryStatistics::getVulnerabilities, + SecurityStandardCategoryStatistics::getToReviewSecurityHotspots, + SecurityStandardCategoryStatistics::getReviewedSecurityHotspots) + .containsExactlyInAnyOrder(0L, 1L, 0L); + assertThat(findRuleInCweByYear(cwe2020, "862")).isNotNull() + .extracting(SecurityStandardCategoryStatistics::getVulnerabilities, + SecurityStandardCategoryStatistics::getToReviewSecurityHotspots, + SecurityStandardCategoryStatistics::getReviewedSecurityHotspots) + .containsExactlyInAnyOrder(1L, 0L, 0L); + assertThat(findRuleInCweByYear(cwe2020, "999")).isNull(); + } + + @Test + public void getCWETop25Report_aggregation_on_portfolio() { + ComponentDto application = db.components().insertPrivateApplication(); + ComponentDto project1 = db.components().insertPrivateProject(); + ComponentDto project2 = db.components().insertPrivateProject(); + + indexIssues( + newDoc("openvul1", project1).setCwe(asList("119")).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_OPEN) + .setSeverity(Severity.MAJOR), + newDoc("openvul2", project2).setCwe(asList("119")).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_REOPENED) + .setSeverity(Severity.MINOR), + newDoc("toreviewhotspot", project1).setCwe(asList("89")).setType(RuleType.SECURITY_HOTSPOT) + .setStatus(Issue.STATUS_TO_REVIEW), + newDoc("only2020", project2).setCwe(asList("862")).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_REOPENED) + .setSeverity(Severity.MINOR), + newDoc("unknown", project2).setCwe(asList("999")).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_REOPENED) + .setSeverity(Severity.MINOR)); + + indexView(application.uuid(), asList(project1.uuid(), project2.uuid())); + + List cweTop25Reports = underTest.getCweTop25Reports(application.uuid(), true); + + SecurityStandardCategoryStatistics cwe2019 = cweTop25Reports.get(0); + assertThat(cwe2019.getChildren()).hasSize(25); + assertThat(findRuleInCweByYear(cwe2019, "119")).isNotNull() + .extracting(SecurityStandardCategoryStatistics::getVulnerabilities, + SecurityStandardCategoryStatistics::getToReviewSecurityHotspots, + SecurityStandardCategoryStatistics::getReviewedSecurityHotspots) + .containsExactlyInAnyOrder(2L, 0L, 0L); + assertThat(findRuleInCweByYear(cwe2019, "89")).isNotNull() + .extracting(SecurityStandardCategoryStatistics::getVulnerabilities, + SecurityStandardCategoryStatistics::getToReviewSecurityHotspots, + SecurityStandardCategoryStatistics::getReviewedSecurityHotspots) + .containsExactlyInAnyOrder(0L, 1L, 0L); + assertThat(findRuleInCweByYear(cwe2019, "862")).isNull(); + assertThat(findRuleInCweByYear(cwe2019, "999")).isNull(); + + SecurityStandardCategoryStatistics cwe2020 = cweTop25Reports.get(1); + assertThat(cwe2020.getChildren()).hasSize(25); + assertThat(findRuleInCweByYear(cwe2020, "119")).isNotNull() + .extracting(SecurityStandardCategoryStatistics::getVulnerabilities, + SecurityStandardCategoryStatistics::getToReviewSecurityHotspots, + SecurityStandardCategoryStatistics::getReviewedSecurityHotspots) + .containsExactlyInAnyOrder(2L, 0L, 0L); + assertThat(findRuleInCweByYear(cwe2020, "89")).isNotNull() + .extracting(SecurityStandardCategoryStatistics::getVulnerabilities, + SecurityStandardCategoryStatistics::getToReviewSecurityHotspots, + SecurityStandardCategoryStatistics::getReviewedSecurityHotspots) + .containsExactlyInAnyOrder(0L, 1L, 0L); + assertThat(findRuleInCweByYear(cwe2020, "862")).isNotNull() + .extracting(SecurityStandardCategoryStatistics::getVulnerabilities, + SecurityStandardCategoryStatistics::getToReviewSecurityHotspots, + SecurityStandardCategoryStatistics::getReviewedSecurityHotspots) + .containsExactlyInAnyOrder(1L, 0L, 0L); + assertThat(findRuleInCweByYear(cwe2020, "999")).isNull(); + } + + private SecurityStandardCategoryStatistics findRuleInCweByYear(SecurityStandardCategoryStatistics statistics, String cweId) { + return statistics.getChildren().stream().filter(stat -> stat.getCategory().equals(cweId)).findAny().orElse(null); + } + private void indexIssues(IssueDoc... issues) { issueIndexer.index(asList(issues).iterator()); authorizationIndexer.allow(stream(issues).map(issue -> new IndexPermissions(issue.projectUuid(), PROJECT).allowAnyone()).collect(toList()));