SONAR-10980 Index security standards in ES and update issues/search WS

8 files changed, 198 insertions, 4 deletions

diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/issue/IssueQuery.java b/server/sonar-server-common/src/main/java/org/sonar/server/issue/IssueQuery.java
index 33ce0641785..86b3400e411 100644
--- a/server/sonar-server-common/src/main/java/org/sonar/server/issue/IssueQuery.java
+++ b/server/sonar-server-common/src/main/java/org/sonar/server/issue/IssueQuery.java
@@ -74,6 +74,9 @@ public class IssueQuery {
   private final Collection<String> languages;
   private final Collection<String> tags;
   private final Collection<String> types;
+  private final Collection<String> owaspTop10;
+  private final Collection<String> sansTop25;
+  private final Collection<String> cwe;
   private final Map<String, PeriodStart> createdAfterByProjectUuids;
   private final Boolean onComponentOnly;
   private final Boolean assigned;
@@ -107,6 +110,9 @@ public class IssueQuery {
     this.languages = defaultCollection(builder.languages);
     this.tags = defaultCollection(builder.tags);
     this.types = defaultCollection(builder.types);
+    this.owaspTop10 = defaultCollection(builder.owaspTop10);
+    this.sansTop25 = defaultCollection(builder.sansTop25);
+    this.cwe = defaultCollection(builder.cwe);
     this.createdAfterByProjectUuids = defaultMap(builder.createdAfterByProjectUuids);
     this.onComponentOnly = builder.onComponentOnly;
     this.assigned = builder.assigned;
@@ -191,6 +197,18 @@ public class IssueQuery {
     return types;
   }
 
+  public Collection<String> owaspTop10() {
+    return owaspTop10;
+  }
+
+  public Collection<String> sansTop25() {
+    return sansTop25;
+  }
+
+  public Collection<String> cwe() {
+    return cwe;
+  }
+
   public Map<String, PeriodStart> createdAfterByProjectUuids() {
     return createdAfterByProjectUuids;
   }
@@ -284,6 +302,9 @@ public class IssueQuery {
     private Collection<String> languages;
     private Collection<String> tags;
     private Collection<String> types;
+    private Collection<String> owaspTop10;
+    private Collection<String> sansTop25;
+    private Collection<String> cwe;
     private Map<String, PeriodStart> createdAfterByProjectUuids;
     private Boolean onComponentOnly = false;
     private Boolean assigned = null;
@@ -388,6 +409,21 @@ public class IssueQuery {
       return this;
     }
 
+    public Builder owaspTop10(@Nullable Collection<String> o) {
+      this.owaspTop10 = o;
+      return this;
+    }
+
+    public Builder sansTop25(@Nullable Collection<String> s) {
+      this.sansTop25 = s;
+      return this;
+    }
+
+    public Builder cwe(@Nullable Collection<String> cwe) {
+      this.cwe = cwe;
+      return this;
+    }
+
     public Builder createdAfterByProjectUuids(@Nullable Map<String, PeriodStart> createdAfterByProjectUuids) {
       this.createdAfterByProjectUuids = createdAfterByProjectUuids;
       return this;
diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/issue/SearchRequest.java b/server/sonar-server-common/src/main/java/org/sonar/server/issue/SearchRequest.java
index b3fb9e7a792..c421627ca7b 100644
--- a/server/sonar-server-common/src/main/java/org/sonar/server/issue/SearchRequest.java
+++ b/server/sonar-server-common/src/main/java/org/sonar/server/issue/SearchRequest.java
@@ -64,6 +64,9 @@ public class SearchRequest {
   private List<String> statuses;
   private List<String> tags;
   private List<String> types;
+  private List<String> owaspTop10;
+  private List<String> sansTop25;
+  private List<String> cwe;
 
   @CheckForNull
   public List<String> getActionPlans() {
@@ -406,6 +409,36 @@ public class SearchRequest {
   }
 
   @CheckForNull
+  public List<String> getOwaspTop10() {
+    return owaspTop10;
+  }
+
+  public SearchRequest setOwaspTop10(@Nullable List<String> owaspTop10) {
+    this.owaspTop10 = owaspTop10;
+    return this;
+  }
+
+  @CheckForNull
+  public List<String> getSansTop25() {
+    return sansTop25;
+  }
+
+  public SearchRequest setSansTop25(@Nullable List<String> sansTop25) {
+    this.sansTop25 = sansTop25;
+    return this;
+  }
+
+  @CheckForNull
+  public List<String> getCwe() {
+    return cwe;
+  }
+
+  public SearchRequest setCwe(@Nullable List<String> cwe) {
+    this.cwe = cwe;
+    return this;
+  }
+
+  @CheckForNull
   public List<String> getComponentRootUuids() {
     return componentRootUuids;
   }
diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueDoc.java b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueDoc.java
index 88fc40c7a93..85ee703388a 100644
--- a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueDoc.java
+++ b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueDoc.java
@@ -290,4 +290,34 @@ public class IssueDoc extends BaseDoc {
     return this;
   }
 
+  @CheckForNull
+  public Collection<String> getOwaspTop10() {
+    return getNullableField(IssueIndexDefinition.FIELD_ISSUE_OWASP_TOP_10);
+  }
+
+  public IssueDoc setOwaspTop10(@Nullable Collection<String> o) {
+    setField(IssueIndexDefinition.FIELD_ISSUE_OWASP_TOP_10, o);
+    return this;
+  }
+
+  @CheckForNull
+  public Collection<String> getSansTop25() {
+    return getNullableField(IssueIndexDefinition.FIELD_ISSUE_SANS_TOP_25);
+  }
+
+  public IssueDoc setSansTop25(@Nullable Collection<String> s) {
+    setField(IssueIndexDefinition.FIELD_ISSUE_SANS_TOP_25, s);
+    return this;
+  }
+
+  @CheckForNull
+  public Collection<String> getCwe() {
+    return getNullableField(IssueIndexDefinition.FIELD_ISSUE_CWE);
+  }
+
+  public IssueDoc setCwe(@Nullable Collection<String> c) {
+    setField(IssueIndexDefinition.FIELD_ISSUE_CWE, c);
+    return this;
+  }
+
 }
diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndex.java
index 6c015fba1ff..9f1bc7382e1 100644
--- a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndex.java
+++ b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndex.java
@@ -104,14 +104,17 @@ import static org.sonarqube.ws.client.issue.IssuesWsParameters.FACET_MODE_EFFORT
 import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_ASSIGNEES;
 import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_AUTHORS;
 import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_CREATED_AT;
+import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_CWE;
 import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_DIRECTORIES;
 import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_FILE_UUIDS;
 import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_LANGUAGES;
 import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_MODULE_UUIDS;
+import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_OWASP_TOP_10;
 import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_PROJECT_UUIDS;
 import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_REPORTERS;
 import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_RESOLUTIONS;
 import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_RULES;
+import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_SANS_TOP_25;
 import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_SEVERITIES;
 import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_STATUSES;
 import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_TAGS;
@@ -140,6 +143,9 @@ public class IssueIndex {
     PARAM_LANGUAGES,
     PARAM_TAGS,
     PARAM_TYPES,
+    PARAM_OWASP_TOP_10,
+    PARAM_SANS_TOP_25,
+    PARAM_CWE,
     PARAM_CREATED_AT);
   public static final String AGGREGATION_NAME_FOR_TAGS = "tags__issues";
   private static final String SUBSTRING_MATCH_REGEXP = ".*%s.*";
@@ -522,6 +528,15 @@ public class IssueIndex {
       if (options.getFacets().contains(PARAM_TYPES)) {
         esSearch.addAggregation(stickyFacetBuilder.buildStickyFacet(IssueIndexDefinition.FIELD_ISSUE_TYPE, PARAM_TYPES, query.types().toArray()));
       }
+      if (options.getFacets().contains(PARAM_OWASP_TOP_10)) {
+        esSearch.addAggregation(stickyFacetBuilder.buildStickyFacet(IssueIndexDefinition.FIELD_ISSUE_OWASP_TOP_10, PARAM_OWASP_TOP_10, query.owaspTop10().toArray()));
+      }
+      if (options.getFacets().contains(PARAM_SANS_TOP_25)) {
+        esSearch.addAggregation(stickyFacetBuilder.buildStickyFacet(IssueIndexDefinition.FIELD_ISSUE_SANS_TOP_25, PARAM_SANS_TOP_25, query.sansTop25().toArray()));
+      }
+      if (options.getFacets().contains(PARAM_CWE)) {
+        esSearch.addAggregation(stickyFacetBuilder.buildStickyFacet(IssueIndexDefinition.FIELD_ISSUE_CWE, PARAM_CWE, query.cwe().toArray()));
+      }
       if (options.getFacets().contains(PARAM_RESOLUTIONS)) {
         esSearch.addAggregation(createResolutionFacet(query, filters, esQuery));
       }
diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java
index c5100bb5698..dd13a01395a 100644
--- a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java
+++ b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java
@@ -92,6 +92,9 @@ public class IssueIndexDefinition implements IndexDefinition {
   public static final String FIELD_ISSUE_STATUS = "status";
   public static final String FIELD_ISSUE_TAGS = "tags";
   public static final String FIELD_ISSUE_TYPE = "type";
+  public static final String FIELD_ISSUE_OWASP_TOP_10 = "owaspTop10";
+  public static final String FIELD_ISSUE_SANS_TOP_25 = "sansTop25";
+  public static final String FIELD_ISSUE_CWE = "cwe";
 
   private final Configuration config;
   private final boolean enableSource;
@@ -151,5 +154,8 @@ public class IssueIndexDefinition implements IndexDefinition {
     type.keywordFieldBuilder(FIELD_ISSUE_STATUS).disableNorms().addSubFields(SORTABLE_ANALYZER).build();
     type.keywordFieldBuilder(FIELD_ISSUE_TAGS).disableNorms().build();
     type.keywordFieldBuilder(FIELD_ISSUE_TYPE).disableNorms().build();
+    type.keywordFieldBuilder(FIELD_ISSUE_OWASP_TOP_10).disableNorms().build();
+    type.keywordFieldBuilder(FIELD_ISSUE_SANS_TOP_25).disableNorms().build();
+    type.keywordFieldBuilder(FIELD_ISSUE_CWE).disableNorms().build();
   }
 }
diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIteratorForSingleChunk.java b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIteratorForSingleChunk.java
index 8eba1043603..e2fd803ba2c 100644
--- a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIteratorForSingleChunk.java
+++ b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIteratorForSingleChunk.java
@@ -21,13 +21,18 @@ package org.sonar.server.issue.index;
 
 import com.google.common.base.CharMatcher;
 import com.google.common.base.Splitter;
-import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.Maps;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import javax.annotation.CheckForNull;
@@ -41,8 +46,14 @@ import org.sonar.db.DbSession;
 import org.sonar.db.ResultSetIterator;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static java.util.Arrays.asList;
+import static java.util.stream.Collectors.toList;
 import static org.sonar.api.utils.DateUtils.longToDate;
 import static org.sonar.db.DatabaseUtils.getLong;
+import static org.sonar.server.issue.ws.SearchAction.SANS_TOP_25_INSECURE_INTERACTION;
+import static org.sonar.server.issue.ws.SearchAction.SANS_TOP_25_POROUS_DEFENSES;
+import static org.sonar.server.issue.ws.SearchAction.SANS_TOP_25_RISKY_RESOURCE;
+import static org.sonar.server.issue.ws.SearchAction.UNKNOWN_STANDARD;
 
 /**
  * Scrolls over table ISSUES and reads documents to populate
@@ -73,11 +84,12 @@ class IssueIteratorForSingleChunk implements IssueIterator {
     "c.scope",
     "c.organization_uuid",
     "c.project_uuid",
+    "c.main_branch_project_uuid",
 
     // column 21
-    "c.main_branch_project_uuid",
     "i.tags",
-    "i.issue_type"
+    "i.issue_type",
+    "r.security_standards"
   };
 
   private static final String SQL_ALL = "select " + StringUtils.join(FIELDS, ",") + " from issues i " +
@@ -89,7 +101,19 @@ class IssueIteratorForSingleChunk implements IssueIterator {
   private static final String ISSUE_KEY_FILTER_SUFFIX = ")";
 
   static final Splitter TAGS_SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings();
+  static final Splitter SECURITY_STANDARDS_SPLITTER = TAGS_SPLITTER;
   static final Splitter MODULE_PATH_SPLITTER = Splitter.on('.').trimResults().omitEmptyStrings();
+  private static final String OWASP_TOP10_PREFIX = "owaspTop10:";
+  private static final String CWE_PREFIX = "cwe:";
+
+  // See https://www.sans.org/top25-software-errors
+  private static final Set<String> INSECURE_CWE = new HashSet<>(asList("89", "78", "79", "434", "352", "601"));
+  private static final Set<String> RISKY_CWE = new HashSet<>(asList("120", "22", "494", "829", "676", "131", "134", "190"));
+  private static final Set<String> POROUS_CWE = new HashSet<>(asList("306", "862", "798", "311", "807", "250", "863", "732", "327", "307", "759"));
+  private static final Map<String, Set<String>> SANS_TOP_25_CWE_MAPPING = ImmutableMap.of(
+    SANS_TOP_25_INSECURE_INTERACTION, INSECURE_CWE,
+    SANS_TOP_25_RISKY_RESOURCE, RISKY_CWE,
+    SANS_TOP_25_POROUS_DEFENSES, POROUS_CWE);
 
   private final DbSession session;
 
@@ -223,11 +247,27 @@ class IssueIteratorForSingleChunk implements IssueIterator {
         doc.setIsMainBranch(false);
       }
       String tags = rs.getString(21);
-      doc.setTags(ImmutableList.copyOf(IssueIteratorForSingleChunk.TAGS_SPLITTER.split(tags == null ? "" : tags)));
+      doc.setTags(IssueIteratorForSingleChunk.TAGS_SPLITTER.splitToList(tags == null ? "" : tags));
       doc.setType(RuleType.valueOf(rs.getInt(22)));
+      String securityStandards = rs.getString(23);
+
+      List<String> standards = IssueIteratorForSingleChunk.SECURITY_STANDARDS_SPLITTER.splitToList(securityStandards == null ? "" : securityStandards);
+      List<String> owaspTop10 = standards.stream().filter(s -> s.startsWith(OWASP_TOP10_PREFIX)).map(s -> s.substring(OWASP_TOP10_PREFIX.length())).collect(toList());
+      doc.setOwaspTop10(owaspTop10.isEmpty() ? Collections.singletonList(UNKNOWN_STANDARD) : owaspTop10);
+      List<String> cwe = standards.stream().filter(s -> s.startsWith(CWE_PREFIX)).map(s -> s.substring(CWE_PREFIX.length())).collect(toList());
+      doc.setCwe(cwe.isEmpty() ? Collections.singletonList(UNKNOWN_STANDARD) : cwe);
+      doc.setSansTop25(getSansTop25(cwe));
       return doc;
     }
 
+    private static List<String> getSansTop25(List<String> cwe) {
+      return SANS_TOP_25_CWE_MAPPING
+        .keySet()
+        .stream()
+        .filter(k -> cwe.stream().anyMatch(SANS_TOP_25_CWE_MAPPING.get(k)::contains))
+        .collect(toList());
+    }
+
     @CheckForNull
     private static String extractDirPath(@Nullable String filePath, String scope) {
       if (filePath != null) {
diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/issue/IssueQueryTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/issue/IssueQueryTest.java
index 1dbec620265..957de3d743f 100644
--- a/server/sonar-server-common/src/test/java/org/sonar/server/issue/IssueQueryTest.java
+++ b/server/sonar-server-common/src/test/java/org/sonar/server/issue/IssueQueryTest.java
@@ -52,6 +52,9 @@ public class IssueQueryTest {
       .languages(newArrayList("xoo"))
       .tags(newArrayList("tag1", "tag2"))
       .types(newArrayList("RELIABILITY", "SECURITY"))
+      .owaspTop10(newArrayList("a1", "a2"))
+      .sansTop25(newArrayList("insecure-interaction", "porous-defenses"))
+      .cwe(newArrayList("12", "125"))
       .organizationUuid("orga")
       .branchUuid("my_branch")
       .createdAfterByProjectUuids(ImmutableMap.of("PROJECT", filterDate))
@@ -74,6 +77,9 @@ public class IssueQueryTest {
     assertThat(query.languages()).containsOnly("xoo");
     assertThat(query.tags()).containsOnly("tag1", "tag2");
     assertThat(query.types()).containsOnly("RELIABILITY", "SECURITY");
+    assertThat(query.owaspTop10()).containsOnly("a1", "a2");
+    assertThat(query.sansTop25()).containsOnly("insecure-interaction", "porous-defenses");
+    assertThat(query.cwe()).containsOnly("12", "125");
     assertThat(query.organizationUuid()).isEqualTo("orga");
     assertThat(query.branchUuid()).isEqualTo("my_branch");
     assertThat(query.createdAfterByProjectUuids()).containsOnly(entry("PROJECT", filterDate));
@@ -128,6 +134,9 @@ public class IssueQueryTest {
       .languages(null)
       .tags(null)
       .types(null)
+      .owaspTop10(null)
+      .sansTop25(null)
+      .cwe(null)
       .createdAfterByProjectUuids(null)
       .build();
     assertThat(query.issueKeys()).isEmpty();
@@ -142,6 +151,9 @@ public class IssueQueryTest {
     assertThat(query.languages()).isEmpty();
     assertThat(query.tags()).isEmpty();
     assertThat(query.types()).isEmpty();
+    assertThat(query.owaspTop10()).isEmpty();
+    assertThat(query.sansTop25()).isEmpty();
+    assertThat(query.cwe()).isEmpty();
     assertThat(query.createdAfterByProjectUuids()).isEmpty();
   }
 
diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java
index 6cf54630bdb..1b5310ca460 100644
--- a/server/sonar-server-common/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java
+++ b/server/sonar-server-common/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java
@@ -22,6 +22,7 @@ package org.sonar.server.issue.index;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Date;
+import java.util.HashSet;
 import java.util.List;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
@@ -56,6 +57,8 @@ import static org.junit.rules.ExpectedException.none;
 import static org.sonar.db.component.ComponentTesting.newFileDto;
 import static org.sonar.server.issue.IssueDocTesting.newDoc;
 import static org.sonar.server.issue.index.IssueIndexDefinition.INDEX_TYPE_ISSUE;
+import static org.sonar.server.issue.ws.SearchAction.SANS_TOP_25_POROUS_DEFENSES;
+import static org.sonar.server.issue.ws.SearchAction.UNKNOWN_STANDARD;
 import static org.sonar.server.permission.index.AuthorizationTypeSupport.TYPE_AUTHORIZATION;
 
 public class IssueIndexerTest {
@@ -134,6 +137,25 @@ public class IssueIndexerTest {
     assertThat(doc.line()).isEqualTo(issue.getLine());
     // functional date
     assertThat(doc.updateDate()).isEqualToIgnoringMillis(new Date(issue.getIssueUpdateTime()));
+    assertThat(doc.getCwe()).containsExactlyInAnyOrder(UNKNOWN_STANDARD);
+    assertThat(doc.getOwaspTop10()).containsExactlyInAnyOrder(UNKNOWN_STANDARD);
+    assertThat(doc.getSansTop25()).isEmpty();
+  }
+
+  @Test
+  public void verify_security_standards_indexation() {
+    RuleDefinitionDto rule = db.rules().insert(r -> r.setSecurityStandards(new HashSet<>(Arrays.asList("cwe:123,owaspTop10:a3,cwe:863"))));
+    ComponentDto project = db.components().insertPrivateProject(organization);
+    ComponentDto dir = db.components().insertComponent(ComponentTesting.newDirectory(project, "src/main/java/foo"));
+    ComponentDto file = db.components().insertComponent(newFileDto(project, dir, "F1"));
+    IssueDto issue = db.issues().insertIssue(IssueTesting.newIssue(rule, project, file));
+
+    underTest.indexOnStartup(emptySet());
+
+    IssueDoc doc = es.getDocuments(INDEX_TYPE_ISSUE, IssueDoc.class).get(0);
+    assertThat(doc.getCwe()).containsExactlyInAnyOrder("123", "863");
+    assertThat(doc.getOwaspTop10()).containsExactlyInAnyOrder("a3");
+    assertThat(doc.getSansTop25()).containsExactlyInAnyOrder(SANS_TOP_25_POROUS_DEFENSES);
   }
 
   @Test