SONAR-14442 Elasticsearch query for CWE Top 25

diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/security/SecurityStandards.java b/server/sonar-server-common/src/main/java/org/sonar/server/security/SecurityStandards.java
index dfa8ad11a96855a49f1ee1c05a8fd00ae6b58f15..22f10f78d5a6ccc8ee0d6db0c6158e4c63844e96 100644 (file)
--- a/server/sonar-server-common/src/main/java/org/sonar/server/security/SecurityStandards.java
+++ b/server/sonar-server-common/src/main/java/org/sonar/server/security/SecurityStandards.java
@@ -22,8 +22,10 @@ package org.sonar.server.security;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Ordering;
+
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -70,18 +72,18 @@ public final class SecurityStandards {
     SANS_TOP_25_POROUS_DEFENSES, POROUS_CWE);
 
   // https://cwe.mitre.org/top25/archive/2019/2019_cwe_top25.html
-  private static final Set<String> CWE_TOP25_2019 = new HashSet<>(
-    asList("119", "79", "20", "200", "125", "89", "416", "190", "352",
+  public static final List<String> CWE_TOP25_2019 =
+    Collections.unmodifiableList(asList("119", "79", "20", "200", "125", "89", "416", "190", "352",
       "22", "78", "787", "287", "476", "732", "434", "611", "94",
       "798", "400", "772", "426", "502", "269", "295"));
 
   // https://cwe.mitre.org/top25/archive/2020/2020_cwe_top25.html
-  private static final Set<String> CWE_TOP25_2020 = new HashSet<>(
-    asList("79", "787", "20", "125", "119", "89", "200", "416", "352",
+  public static final List<String> CWE_TOP25_2020 =
+    Collections.unmodifiableList(asList("79", "787", "20", "125", "119", "89", "200", "416", "352",
       "78", "190", "22", "476", "287", "434", "732", "94", "522",
       "611", "798", "502", "269", "400", "306", "862"));
 
-  public static final Map<String, Set<String>> CWES_BY_CWE_TOP_25 = ImmutableMap.of(
+  public static final Map<String, List<String>> CWES_BY_CWE_TOP_25 = ImmutableMap.of(
     "2019", CWE_TOP25_2019,
     "2020", CWE_TOP25_2020);
 

diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java
index 22008fa3dec0cdc65cfca976ea4fbeece17be21e..a948489caab5a0c4792c9d0893f1fbce001203f5 100644 (file)
--- a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java
+++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java
@@ -33,6 +33,7 @@ import java.util.OptionalInt;
 import java.util.OptionalLong;
 import java.util.Set;
 import java.util.function.Consumer;
+import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 import javax.annotation.CheckForNull;
@@ -166,6 +167,7 @@ import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_VULN
 import static org.sonar.server.issue.index.IssueIndexDefinition.TYPE_ISSUE;
 import static org.sonar.server.security.SecurityReviewRating.computePercent;
 import static org.sonar.server.security.SecurityReviewRating.computeRating;
+import static org.sonar.server.security.SecurityStandards.CWES_BY_CWE_TOP_25;
 import static org.sonar.server.security.SecurityStandards.SANS_TOP_25_INSECURE_INTERACTION;
 import static org.sonar.server.security.SecurityStandards.SANS_TOP_25_POROUS_DEFENSES;
 import static org.sonar.server.security.SecurityStandards.SANS_TOP_25_RISKY_RESOURCE;
@@ -1016,19 +1018,32 @@ public class IssueIndex {
       .forEach(sansCategory -> request.aggregation(newSecurityReportSubAggregations(
         AggregationBuilders.filter(sansCategory, boolQuery().filter(termQuery(FIELD_ISSUE_SANS_TOP_25, sansCategory))),
         includeCwe,
-        Optional.ofNullable(SecurityStandards.CWES_BY_SANS_TOP_25.get(sansCategory)))));
+        SecurityStandards.CWES_BY_SANS_TOP_25.get(sansCategory))));
     return processSecurityReportSearchResults(request, includeCwe);
   }
 
-  public List<SecurityStandardCategoryStatistics> getCweTop25Reports(String uuid, boolean isViewOrApp) {
-    // TODO:: Mock data - SONAR-14447 elasticsearch query
-    return Arrays.asList(
-      new SecurityStandardCategoryStatistics("2019", 1, OptionalInt.empty(), 10, 5, 10,
-        SecurityStandards.CWES_BY_CWE_TOP_25.get("2019").stream().map(cwe -> new SecurityStandardCategoryStatistics(cwe, 1, OptionalInt.empty(), 1, 3, 2, null))
-          .collect(toList())),
-      new SecurityStandardCategoryStatistics("2020", 0, OptionalInt.empty(), 9, 5, 10,
-        SecurityStandards.CWES_BY_CWE_TOP_25.get("2020").stream().map(cwe -> new SecurityStandardCategoryStatistics(cwe, 1, OptionalInt.empty(), 1, 3, 4, null))
-          .collect(toList())));
+  public List<SecurityStandardCategoryStatistics> getCweTop25Reports(String projectUuid, boolean isViewOrApp) {
+    SearchSourceBuilder request = prepareNonClosedVulnerabilitiesAndHotspotSearch(projectUuid, isViewOrApp);
+    CWES_BY_CWE_TOP_25.keySet()
+      .forEach(cweYear -> request.aggregation(
+        newSecurityReportSubAggregations(
+          AggregationBuilders.filter(cweYear, boolQuery().filter(existsQuery(FIELD_ISSUE_CWE))),
+          true,
+          CWES_BY_CWE_TOP_25.get(cweYear))));
+    List<SecurityStandardCategoryStatistics> result = processSecurityReportSearchResults(request, true);
+    for (SecurityStandardCategoryStatistics cweReport : result) {
+      Set<String> foundRules = cweReport.getChildren().stream()
+        .map(SecurityStandardCategoryStatistics::getCategory)
+        .collect(Collectors.toSet());
+      CWES_BY_CWE_TOP_25.get(cweReport.getCategory()).stream()
+        .filter(rule -> !foundRules.contains(rule))
+        .forEach(rule -> cweReport.getChildren().add(emptyCweStatistics(rule)));
+    }
+    return result;
+  }
+
+  private static SecurityStandardCategoryStatistics emptyCweStatistics(String rule) {
+    return new SecurityStandardCategoryStatistics(rule, 0, OptionalInt.of(1), 0, 0, 1, null);
   }
 
   public List<SecurityStandardCategoryStatistics> getSonarSourceReport(String projectUuid, boolean isViewOrApp, boolean includeCwe) {
@@ -1038,7 +1053,7 @@ public class IssueIndex {
         newSecurityReportSubAggregations(
           AggregationBuilders.filter(sonarsourceCategory.getKey(), boolQuery().filter(termQuery(FIELD_ISSUE_SQ_SECURITY_CATEGORY, sonarsourceCategory.getKey()))),
           includeCwe,
-          Optional.ofNullable(SecurityStandards.CWES_BY_SQ_CATEGORY.get(sonarsourceCategory)))));
+          SecurityStandards.CWES_BY_SQ_CATEGORY.get(sonarsourceCategory))));
     return processSecurityReportSearchResults(request, includeCwe);
   }
 
@@ -1049,7 +1064,7 @@ public class IssueIndex {
         newSecurityReportSubAggregations(
           AggregationBuilders.filter(owaspCategory, boolQuery().filter(termQuery(FIELD_ISSUE_OWASP_TOP_10, owaspCategory))),
           includeCwe,
-          Optional.empty())));
+          null)));
     return processSecurityReportSearchResults(request, includeCwe);
   }
 
@@ -1095,16 +1110,16 @@ public class IssueIndex {
       reviewedSecurityHotspots, securityReviewRating, children);
   }
 
-  private static AggregationBuilder newSecurityReportSubAggregations(AggregationBuilder categoriesAggs, boolean includeCwe, Optional<Set<String>> cwesInCategory) {
+  private static AggregationBuilder newSecurityReportSubAggregations(AggregationBuilder categoriesAggs, boolean includeCwe, @Nullable Collection<String> cwesInCategory) {
     AggregationBuilder aggregationBuilder = addSecurityReportIssueCountAggregations(categoriesAggs);
     if (includeCwe) {
       final TermsAggregationBuilder cwesAgg = AggregationBuilders.terms(AGG_CWES)
         .field(FIELD_ISSUE_CWE)
         // 100 should be enough to display all CWEs. If not, the UI will be broken anyway
         .size(MAX_FACET_SIZE);
-      cwesInCategory.ifPresent(set -> {
-        cwesAgg.includeExclude(new IncludeExclude(set.toArray(new String[0]), new String[0]));
-      });
+      if (cwesInCategory != null) {
+        cwesAgg.includeExclude(new IncludeExclude(cwesInCategory.toArray(new String[0]), new String[0]));
+      }
       categoriesAggs.subAggregation(addSecurityReportIssueCountAggregations(cwesAgg));
     }
     return aggregationBuilder;

diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityReportsTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityReportsTest.java
index a36fd41da89dcc6cea1288087309d10f438b9d23..58b4448e9a7637c2a7668fdd6eb1562cdfac5142 100644 (file)
--- a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityReportsTest.java
+++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityReportsTest.java
@@ -23,6 +23,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.OptionalInt;
 import java.util.stream.Collectors;
+
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -294,6 +295,120 @@ public class IssueIndexSecurityReportsTest {
     assertThat(sansTop25Report).allMatch(category -> category.getChildren().isEmpty());
   }
 
+  @Test
+  public void getCWETop25Report_aggregation() {
+    ComponentDto project = newPrivateProjectDto();
+    indexIssues(
+      newDoc("openvul", project).setCwe(asList("119")).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_OPEN)
+        .setSeverity(Severity.MAJOR),
+      newDoc("notopenvul", project).setCwe(asList("119")).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_CLOSED)
+        .setResolution(Issue.RESOLUTION_FIXED)
+        .setSeverity(Severity.BLOCKER),
+      newDoc("toreviewhotspot", project).setCwe(asList("89")).setType(RuleType.SECURITY_HOTSPOT)
+        .setStatus(Issue.STATUS_TO_REVIEW),
+      newDoc("only2020", project).setCwe(asList("862")).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_REOPENED)
+        .setSeverity(Severity.MINOR),
+      newDoc("unknown", project).setCwe(asList("999")).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_REOPENED)
+        .setSeverity(Severity.MINOR));
+
+    List<SecurityStandardCategoryStatistics> cweTop25Reports = underTest.getCweTop25Reports(project.uuid(), false);
+
+    SecurityStandardCategoryStatistics cwe2019 = cweTop25Reports.get(0);
+    assertThat(cwe2019.getChildren()).hasSize(25);
+    assertThat(findRuleInCweByYear(cwe2019, "119")).isNotNull()
+      .extracting(SecurityStandardCategoryStatistics::getVulnerabilities,
+        SecurityStandardCategoryStatistics::getToReviewSecurityHotspots,
+        SecurityStandardCategoryStatistics::getReviewedSecurityHotspots)
+      .containsExactlyInAnyOrder(1L, 0L, 0L);
+    assertThat(findRuleInCweByYear(cwe2019, "89")).isNotNull()
+      .extracting(SecurityStandardCategoryStatistics::getVulnerabilities,
+        SecurityStandardCategoryStatistics::getToReviewSecurityHotspots,
+        SecurityStandardCategoryStatistics::getReviewedSecurityHotspots)
+      .containsExactlyInAnyOrder(0L, 1L, 0L);
+    assertThat(findRuleInCweByYear(cwe2019, "862")).isNull();
+    assertThat(findRuleInCweByYear(cwe2019, "999")).isNull();
+
+    SecurityStandardCategoryStatistics cwe2020 = cweTop25Reports.get(1);
+    assertThat(cwe2020.getChildren()).hasSize(25);
+    assertThat(findRuleInCweByYear(cwe2020, "119")).isNotNull()
+      .extracting(SecurityStandardCategoryStatistics::getVulnerabilities,
+        SecurityStandardCategoryStatistics::getToReviewSecurityHotspots,
+        SecurityStandardCategoryStatistics::getReviewedSecurityHotspots)
+      .containsExactlyInAnyOrder(1L, 0L, 0L);
+    assertThat(findRuleInCweByYear(cwe2020, "89")).isNotNull()
+      .extracting(SecurityStandardCategoryStatistics::getVulnerabilities,
+        SecurityStandardCategoryStatistics::getToReviewSecurityHotspots,
+        SecurityStandardCategoryStatistics::getReviewedSecurityHotspots)
+      .containsExactlyInAnyOrder(0L, 1L, 0L);
+    assertThat(findRuleInCweByYear(cwe2020, "862")).isNotNull()
+      .extracting(SecurityStandardCategoryStatistics::getVulnerabilities,
+        SecurityStandardCategoryStatistics::getToReviewSecurityHotspots,
+        SecurityStandardCategoryStatistics::getReviewedSecurityHotspots)
+      .containsExactlyInAnyOrder(1L, 0L, 0L);
+    assertThat(findRuleInCweByYear(cwe2020, "999")).isNull();
+  }
+
+  @Test
+  public void getCWETop25Report_aggregation_on_portfolio() {
+    ComponentDto application = db.components().insertPrivateApplication();
+    ComponentDto project1 = db.components().insertPrivateProject();
+    ComponentDto project2 = db.components().insertPrivateProject();
+
+    indexIssues(
+      newDoc("openvul1", project1).setCwe(asList("119")).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_OPEN)
+        .setSeverity(Severity.MAJOR),
+      newDoc("openvul2", project2).setCwe(asList("119")).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_REOPENED)
+        .setSeverity(Severity.MINOR),
+      newDoc("toreviewhotspot", project1).setCwe(asList("89")).setType(RuleType.SECURITY_HOTSPOT)
+        .setStatus(Issue.STATUS_TO_REVIEW),
+      newDoc("only2020", project2).setCwe(asList("862")).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_REOPENED)
+        .setSeverity(Severity.MINOR),
+      newDoc("unknown", project2).setCwe(asList("999")).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_REOPENED)
+        .setSeverity(Severity.MINOR));
+
+    indexView(application.uuid(), asList(project1.uuid(), project2.uuid()));
+
+    List<SecurityStandardCategoryStatistics> cweTop25Reports = underTest.getCweTop25Reports(application.uuid(), true);
+
+    SecurityStandardCategoryStatistics cwe2019 = cweTop25Reports.get(0);
+    assertThat(cwe2019.getChildren()).hasSize(25);
+    assertThat(findRuleInCweByYear(cwe2019, "119")).isNotNull()
+      .extracting(SecurityStandardCategoryStatistics::getVulnerabilities,
+        SecurityStandardCategoryStatistics::getToReviewSecurityHotspots,
+        SecurityStandardCategoryStatistics::getReviewedSecurityHotspots)
+      .containsExactlyInAnyOrder(2L, 0L, 0L);
+    assertThat(findRuleInCweByYear(cwe2019, "89")).isNotNull()
+      .extracting(SecurityStandardCategoryStatistics::getVulnerabilities,
+        SecurityStandardCategoryStatistics::getToReviewSecurityHotspots,
+        SecurityStandardCategoryStatistics::getReviewedSecurityHotspots)
+      .containsExactlyInAnyOrder(0L, 1L, 0L);
+    assertThat(findRuleInCweByYear(cwe2019, "862")).isNull();
+    assertThat(findRuleInCweByYear(cwe2019, "999")).isNull();
+
+    SecurityStandardCategoryStatistics cwe2020 = cweTop25Reports.get(1);
+    assertThat(cwe2020.getChildren()).hasSize(25);
+    assertThat(findRuleInCweByYear(cwe2020, "119")).isNotNull()
+      .extracting(SecurityStandardCategoryStatistics::getVulnerabilities,
+        SecurityStandardCategoryStatistics::getToReviewSecurityHotspots,
+        SecurityStandardCategoryStatistics::getReviewedSecurityHotspots)
+      .containsExactlyInAnyOrder(2L, 0L, 0L);
+    assertThat(findRuleInCweByYear(cwe2020, "89")).isNotNull()
+      .extracting(SecurityStandardCategoryStatistics::getVulnerabilities,
+        SecurityStandardCategoryStatistics::getToReviewSecurityHotspots,
+        SecurityStandardCategoryStatistics::getReviewedSecurityHotspots)
+      .containsExactlyInAnyOrder(0L, 1L, 0L);
+    assertThat(findRuleInCweByYear(cwe2020, "862")).isNotNull()
+      .extracting(SecurityStandardCategoryStatistics::getVulnerabilities,
+        SecurityStandardCategoryStatistics::getToReviewSecurityHotspots,
+        SecurityStandardCategoryStatistics::getReviewedSecurityHotspots)
+      .containsExactlyInAnyOrder(1L, 0L, 0L);
+    assertThat(findRuleInCweByYear(cwe2020, "999")).isNull();
+  }
+
+  private SecurityStandardCategoryStatistics findRuleInCweByYear(SecurityStandardCategoryStatistics statistics, String cweId) {
+    return statistics.getChildren().stream().filter(stat -> stat.getCategory().equals(cweId)).findAny().orElse(null);
+  }
+
   private void indexIssues(IssueDoc... issues) {
     issueIndexer.index(asList(issues).iterator());
     authorizationIndexer.allow(stream(issues).map(issue -> new IndexPermissions(issue.projectUuid(), PROJECT).allowAnyone()).collect(toList()));