Преглед на файлове

SONAR-11983 new "sonarsource" security report

tags/7.8
Michal Duda преди 5 години
родител
ревизия
c240fefe79
променени са 19 файла, в които са добавени 768 реда и са изтрити 115 реда
  1. 10
    0
      server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/es/MigrationEsClient.java
  2. 43
    0
      server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v77/AddSonarsourceSecurityElasticsearchMapping.java
  3. 1
    1
      server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v77/DbVersion77.java
  4. 59
    0
      server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v77/AddSonarsourceSecurityElasticsearchMappingTest.java
  5. 1
    1
      server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v77/DbVersion77Test.java
  6. 9
    0
      server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueDoc.java
  7. 2
    0
      server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java
  8. 2
    0
      server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIteratorForSingleChunk.java
  9. 36
    0
      server/sonar-server-common/src/main/java/org/sonar/server/issue/index/SecurityStandardHelper.java
  10. 0
    23
      server/sonar-server/src/main/java/org/sonar/server/es/IndexCreator.java
  11. 14
    0
      server/sonar-server/src/main/java/org/sonar/server/es/MigrationEsClientImpl.java
  12. 12
    0
      server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java
  13. 41
    26
      server/sonar-server/src/main/java/org/sonar/server/securityreport/ws/ShowAction.java
  14. 0
    64
      server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java
  15. 34
    0
      server/sonar-server/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java
  16. 72
    0
      server/sonar-server/src/test/java/org/sonar/server/securityreport/ws/ShowActionTest.java
  17. 205
    0
      server/sonar-server/src/test/resources/org/sonar/server/securityreport/ws/ShowActionTest/sonarsourceSecurityNoCwe.json
  18. 226
    0
      server/sonar-server/src/test/resources/org/sonar/server/securityreport/ws/ShowActionTest/sonarsourceSecurityWithCwe.json
  19. 1
    0
      sonar-ws/src/main/java/org/sonarqube/ws/client/issue/IssuesWsParameters.java

+ 10
- 0
server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/es/MigrationEsClient.java Целия файл

@@ -25,4 +25,14 @@ public interface MigrationEsClient {
* This method is re-entrant and does not fail if indexName or otherIndexNames do not exist
*/
void deleteIndexes(String name, String... otherNames);

/**
* Adds a new mapping to an existing elasticsearch index
*
* @param index name of the index that the mapping is added to
* @param type document type in the index
* @param mappingName name of the new mapping
* @param mappingType type of the new mapping
*/
void addMappingToExistingIndex(String index, String type, String mappingName, String mappingType);
}

+ 43
- 0
server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v77/AddSonarsourceSecurityElasticsearchMapping.java Целия файл

@@ -0,0 +1,43 @@
/*
* SonarQube
* Copyright (C) 2009-2019 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.platform.db.migration.version.v77;

import java.sql.SQLException;
import org.sonar.db.Database;
import org.sonar.server.platform.db.migration.SupportsBlueGreen;
import org.sonar.server.platform.db.migration.es.MigrationEsClient;
import org.sonar.server.platform.db.migration.step.DdlChange;

@SupportsBlueGreen
public class AddSonarsourceSecurityElasticsearchMapping extends DdlChange {

private final MigrationEsClient migrationEsClient;

public AddSonarsourceSecurityElasticsearchMapping(Database db, MigrationEsClient migrationEsClient) {
super(db);
this.migrationEsClient = migrationEsClient;
}

@Override
public void execute(Context context) throws SQLException {
migrationEsClient.addMappingToExistingIndex("issues", "auth", "sonarsourceSecurity", "keyword");
}

}

+ 1
- 1
server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v77/DbVersion77.java Целия файл

@@ -38,6 +38,6 @@ public class DbVersion77 implements DbVersion {
.add(2609, "Delete exceeding favorites when there are more than 100 for a user", DeleteFavoritesExceedingOneHundred.class)
.add(2610, "Truncate ES_QUEUE table content", TruncateEsQueue.class)
.add(2611, "Add SNAPSHOTS.BUILD_STRING", AddBuildStringToSnapshot.class)
;
.add(2612, "Add 'sonarsourceSecurity' mapping to elasticsearch index 'issues'", AddSonarsourceSecurityElasticsearchMapping.class);
}
}

+ 59
- 0
server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v77/AddSonarsourceSecurityElasticsearchMappingTest.java Целия файл

@@ -0,0 +1,59 @@
/*
* SonarQube
* Copyright (C) 2009-2019 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/

package org.sonar.server.platform.db.migration.version.v77;

import java.sql.SQLException;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.sonar.db.CoreDbTester;
import org.sonar.server.platform.db.migration.es.MigrationEsClient;

import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;

public class AddSonarsourceSecurityElasticsearchMappingTest {

@Rule
public final CoreDbTester db = CoreDbTester.createEmpty();

@Rule
public ExpectedException expectedException = ExpectedException.none();

private MigrationEsClient esClient = mock(MigrationEsClient.class);
private AddSonarsourceSecurityElasticsearchMapping underTest = new AddSonarsourceSecurityElasticsearchMapping(db.database(), esClient);

@Test
public void migration_adds_new_issues_mapping() throws SQLException {
underTest.execute();

verify(esClient).addMappingToExistingIndex("issues", "auth", "sonarsourceSecurity", "keyword");
}

public void migration_is_reentrant() throws SQLException {
underTest.execute();

underTest.execute();

verify(esClient).addMappingToExistingIndex("issues", "auth", "sonarsourceSecurity", "keyword");
}

}

+ 1
- 1
server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v77/DbVersion77Test.java Целия файл

@@ -36,7 +36,7 @@ public class DbVersion77Test {

@Test
public void verify_migration_count() {
verifyMigrationCount(underTest, 11);
verifyMigrationCount(underTest, 12);
}

}

+ 9
- 0
server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueDoc.java Целия файл

@@ -314,4 +314,13 @@ public class IssueDoc extends BaseDoc {
return this;
}

@CheckForNull
public Collection<String> getSonarSourceSecurityCategories() {
return getNullableField(IssueIndexDefinition.FIELD_ISSUE_SONARSOURCE_SECURITY);
}

public IssueDoc setSonarSourceSecurityCategories(@Nullable Collection<String> c) {
setField(IssueIndexDefinition.FIELD_ISSUE_SONARSOURCE_SECURITY, c);
return this;
}
}

+ 2
- 0
server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java Целия файл

@@ -99,6 +99,7 @@ public class IssueIndexDefinition implements IndexDefinition {
public static final String FIELD_ISSUE_OWASP_TOP_10 = "owaspTop10";
public static final String FIELD_ISSUE_SANS_TOP_25 = "sansTop25";
public static final String FIELD_ISSUE_CWE = "cwe";
public static final String FIELD_ISSUE_SONARSOURCE_SECURITY = "sonarsourceSecurity";

private final Configuration config;
private final boolean enableSource;
@@ -159,5 +160,6 @@ public class IssueIndexDefinition implements IndexDefinition {
mapping.keywordFieldBuilder(FIELD_ISSUE_OWASP_TOP_10).disableNorms().build();
mapping.keywordFieldBuilder(FIELD_ISSUE_SANS_TOP_25).disableNorms().build();
mapping.keywordFieldBuilder(FIELD_ISSUE_CWE).disableNorms().build();
mapping.keywordFieldBuilder(FIELD_ISSUE_SONARSOURCE_SECURITY).disableNorms().build();
}
}

+ 2
- 0
server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIteratorForSingleChunk.java Целия файл

@@ -47,6 +47,7 @@ import static org.sonar.server.issue.index.SecurityStandardHelper.getCwe;
import static org.sonar.server.issue.index.SecurityStandardHelper.getOwaspTop10;
import static org.sonar.server.issue.index.SecurityStandardHelper.getSansTop25;
import static org.sonar.server.issue.index.SecurityStandardHelper.getSecurityStandards;
import static org.sonar.server.issue.index.SecurityStandardHelper.getSonarSourceSecurityCategories;

/**
* Scrolls over table ISSUES and reads documents to populate
@@ -237,6 +238,7 @@ class IssueIteratorForSingleChunk implements IssueIterator {
List<String> cwe = getCwe(standards);
doc.setCwe(cwe);
doc.setSansTop25(getSansTop25(cwe));
doc.setSonarSourceSecurityCategories(getSonarSourceSecurityCategories(cwe));
return doc;
}


+ 36
- 0
server/sonar-server-common/src/main/java/org/sonar/server/issue/index/SecurityStandardHelper.java Целия файл

@@ -47,6 +47,30 @@ public class SecurityStandardHelper {
private static final Set<String> RISKY_CWE = new HashSet<>(asList("120", "22", "494", "829", "676", "131", "134", "190"));
private static final Set<String> POROUS_CWE = new HashSet<>(asList("306", "862", "798", "311", "807", "250", "863", "732", "327", "307", "759"));


public static final Map<String, List<String>> SONARSOURCE_CWE_MAPPING = ImmutableMap.<String, List<String>>builder()
.put("sql-injection", asList("89", "564"))
.put("command-injection", asList("78", "77"))
.put("path-traversal-injection", singletonList("22"))
.put("ldap-injection", singletonList("90"))
.put("xpath-injection", singletonList("643"))
.put("expression-lang-injection", singletonList("917"))
.put("rce", singletonList("94"))
.put("dos", singletonList("400"))
.put("ssrf", singletonList("918"))
.put("csrf", singletonList("352"))
.put("xss", asList("79", "80", "81", "82", "83", "84", "85", "86", "87"))
.put("log-injection", singletonList("117"))
.put("http-response-splitting", singletonList("113"))
.put("open-redirect", singletonList("601"))
.put("xxe", asList("611", "827"))
.put("object-injection", singletonList("470"))
.put("weak-cryptography", asList("326", "295", "326", "327", "297", "780", "328", "327"))
.put("auth", asList("798", "640", "620", "549", "522", "521", "263", "262", "261", "259", "522", "284"))
.put("insecure-conf", asList("102", "489"))
.put("file-manipulation", asList("97", "73"))
.build();

public static final Map<String, Set<String>> SANS_TOP_25_CWE_MAPPING = ImmutableMap.of(
SANS_TOP_25_INSECURE_INTERACTION, INSECURE_CWE,
SANS_TOP_25_RISKY_RESOURCE, RISKY_CWE,
@@ -70,6 +94,14 @@ public class SecurityStandardHelper {
.collect(toList());
}

public static List<String> getSonarSourceSecurityCategories(Collection<String> cwe) {
return SONARSOURCE_CWE_MAPPING
.keySet()
.stream()
.filter(k -> cwe.stream().anyMatch(SONARSOURCE_CWE_MAPPING.get(k)::contains))
.collect(toList());
}

public static List<String> getOwaspTop10(Collection<String> securityStandards) {
List<String> result = securityStandards.stream()
.filter(s -> s.startsWith(OWASP_TOP10_PREFIX))
@@ -90,6 +122,10 @@ public class SecurityStandardHelper {
return getSansTop25(getCwe(getSecurityStandards(securityStandards)));
}

public static List<String> getSonarSourceSecurityCategories(String securityStandards) {
return getSonarSourceSecurityCategories(getCwe(getSecurityStandards(securityStandards)));
}

public static List<String> getOwaspTop10(String securityStandards) {
return getOwaspTop10(getSecurityStandards(securityStandards));
}

+ 0
- 23
server/sonar-server/src/main/java/org/sonar/server/es/IndexCreator.java Целия файл

@@ -24,7 +24,6 @@ import java.util.Collection;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
@@ -34,7 +33,6 @@ import org.sonar.api.config.Configuration;
import org.sonar.api.server.ServerSide;
import org.sonar.api.utils.log.Logger;
import org.sonar.api.utils.log.Loggers;
import org.sonar.process.ProcessProperties;
import org.sonar.server.es.metadata.EsDbCompatibility;
import org.sonar.server.es.metadata.MetadataIndex;
import org.sonar.server.es.metadata.MetadataIndexDefinition;
@@ -85,26 +83,13 @@ public class IndexCreator implements Startable {
// create indices that do not exist or that have a new definition (different mapping, cluster enabled, ...)
for (BuiltIndex<?> builtIndex : definitions.getIndices().values()) {
Index index = builtIndex.getMainType().getIndex();
String indexName = index.getName();
boolean exists = client.prepareIndicesExist(index).get().isExists();
if (exists && !builtIndex.getMainType().equals(metadataMainType) && hasDefinitionChange(builtIndex)) {
verifyNotBlueGreenDeployment(indexName);
LOGGER.info("Delete Elasticsearch index {} (structure changed)", indexName);
deleteIndex(indexName);
exists = false;
}
if (!exists) {
createIndex(builtIndex, true);
}
}
}

private void verifyNotBlueGreenDeployment(String indexToBeDeleted) {
if (configuration.getBoolean(ProcessProperties.Property.BLUE_GREEN_ENABLED.getKey()).orElse(false)) {
throw new IllegalStateException("Blue/green deployment is not supported. Elasticsearch index [" + indexToBeDeleted + "] changed and needs to be dropped.");
}
}

@Override
public void stop() {
// nothing to do
@@ -145,14 +130,6 @@ public class IndexCreator implements Startable {
client.nativeClient().admin().indices().prepareDelete(indexName).get();
}

private boolean hasDefinitionChange(BuiltIndex<?> index) {
return metadataIndex.getHash(index.getMainType().getIndex())
.map(hash -> {
String defHash = IndexDefinitionHash.of(index);
return !StringUtils.equals(hash, defHash);
}).orElse(true);
}

private void checkDbCompatibility(Collection<BuiltIndex> definitions) {
List<String> existingIndices = loadExistingIndicesExceptMetadata(definitions);
if (!existingIndices.isEmpty()) {

+ 14
- 0
server/sonar-server/src/main/java/org/sonar/server/es/MigrationEsClientImpl.java Целия файл

@@ -28,6 +28,8 @@ import org.sonar.api.utils.log.Loggers;
import org.sonar.core.util.stream.MoreCollectors;
import org.sonar.server.platform.db.migration.es.MigrationEsClient;

import static java.lang.String.format;

public class MigrationEsClientImpl implements MigrationEsClient {
private final EsClient client;

@@ -45,6 +47,18 @@ public class MigrationEsClientImpl implements MigrationEsClient {
.forEach(this::deleteIndex);
}

@Override
public void addMappingToExistingIndex(String index, String type, String mappingName, String mappingType) {
IndexStats stats = client.nativeClient().admin().indices().prepareStats().get().getIndex(index);
if (stats != null) {
Loggers.get(getClass()).info("Add mapping [{}] to Elasticsearch index [{}]", mappingName, index);
client.nativeClient().admin().indices().preparePutMapping(index)
.setType(type)
.setSource(mappingName, format("type=%s", mappingType))
.get();
}
}

private void deleteIndex(String index) {
Loggers.get(getClass()).info("Drop Elasticsearch index [{}]", index);
client.nativeClient().admin().indices().prepareDelete(index).get();

+ 12
- 0
server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java Целия файл

@@ -145,6 +145,7 @@ import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_RULE
import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_SANS_TOP_25;
import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_SEVERITY;
import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_SEVERITY_VALUE;
import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_SONARSOURCE_SECURITY;
import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_STATUS;
import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_TAGS;
import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_TYPE;
@@ -852,6 +853,17 @@ public class IssueIndex {
return processSecurityReportSearchResults(request, includeCwe);
}

public List<SecurityStandardCategoryStatistics> getSonarSourceReport(String projectUuid, boolean isViewOrApp, boolean includeCwe) {
SearchRequestBuilder request = prepareNonClosedVulnerabilitiesAndHotspotSearch(projectUuid, isViewOrApp);
SecurityStandardHelper.SONARSOURCE_CWE_MAPPING.keySet().forEach(sansCategory -> {
AggregationBuilder sansCategoryAggs = AggregationBuilders
.filter(sansCategory, boolQuery()
.filter(termQuery(FIELD_ISSUE_SONARSOURCE_SECURITY, sansCategory)));
request.addAggregation(addSecurityReportSubAggregations(sansCategoryAggs, includeCwe));
});
return processSecurityReportSearchResults(request, includeCwe);
}

public List<SecurityStandardCategoryStatistics> getOwaspTop10Report(String projectUuid, boolean isViewOrApp, boolean includeCwe) {
SearchRequestBuilder request = prepareNonClosedVulnerabilitiesAndHotspotSearch(projectUuid, isViewOrApp);
Stream.concat(IntStream.rangeClosed(1, 10).mapToObj(i -> "a" + i), Stream.of(UNKNOWN_STANDARD)).forEach(owaspCategory -> {

+ 41
- 26
server/sonar-server/src/main/java/org/sonar/server/securityreport/ws/ShowAction.java Целия файл

@@ -38,6 +38,7 @@ import org.sonar.db.rule.RuleDto;
import org.sonar.server.component.ComponentFinder;
import org.sonar.server.issue.index.IssueIndex;
import org.sonar.server.issue.index.SecurityStandardCategoryStatistics;
import org.sonar.server.issue.index.SecurityStandardHelper;
import org.sonar.server.qualityprofile.QPMeasureData;
import org.sonar.server.qualityprofile.QualityProfile;
import org.sonar.server.user.UserSession;
@@ -54,12 +55,15 @@ import static org.sonar.server.issue.index.SecurityStandardHelper.UNKNOWN_STANDA
import static org.sonar.server.issue.index.SecurityStandardHelper.getCwe;
import static org.sonar.server.issue.index.SecurityStandardHelper.getOwaspTop10;
import static org.sonar.server.issue.index.SecurityStandardHelper.getSansTop25;
import static org.sonar.server.issue.index.SecurityStandardHelper.getSonarSourceSecurityCategories;
import static org.sonar.server.ws.WsUtils.writeProtobuf;
import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_OWASP_TOP_10;
import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_SANS_TOP_25;
import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_SONARSOURCE_SECURITY;

public class ShowAction implements SecurityReportsWsAction {

private static final String UNSUPPORTED_STANDARD_MSG = "Unsupported standard: '%s'";
private static final String PARAM_PROJECT = "project";
private static final String PARAM_BRANCH = "branch";
private static final String PARAM_INCLUDE_DISTRIBUTION = "includeDistribution";
@@ -96,7 +100,7 @@ public class ShowAction implements SecurityReportsWsAction {
.setExampleValue("branch-2.0");
action.createParam(PARAM_STANDARD)
.setDescription("Security standard")
.setPossibleValues(PARAM_OWASP_TOP_10, PARAM_SANS_TOP_25)
.setPossibleValues(PARAM_OWASP_TOP_10, PARAM_SANS_TOP_25, PARAM_SONARSOURCE_SECURITY)
.setRequired(true);
action.createParam(PARAM_INCLUDE_DISTRIBUTION)
.setDescription("To return CWE distribution")
@@ -142,8 +146,13 @@ public class ShowAction implements SecurityReportsWsAction {
completeStatistics(sansTop25Report, projectDto, standard, includeCwe);
writeResponse(request, response, sansTop25Report);
break;
case PARAM_SONARSOURCE_SECURITY:
List<SecurityStandardCategoryStatistics> sonarSourceReport = issueIndex.getSonarSourceReport(projectDto.uuid(), isViewOrApp, includeCwe);
completeStatistics(sonarSourceReport, projectDto, standard, includeCwe);
writeResponse(request, response, sonarSourceReport);
break;
default:
throw new IllegalArgumentException("Unsupported standard: '" + standard + "'");
throw new IllegalArgumentException(String.format(UNSUPPORTED_STANDARD_MSG, standard));
}
}

@@ -157,27 +166,30 @@ public class ShowAction implements SecurityReportsWsAction {
List<OrgActiveRuleDto> activeRuleDtos = dbClient.activeRuleDao().selectByTypeAndProfileUuids(dbSession,
asList(RuleType.SECURITY_HOTSPOT.getDbConstant(), RuleType.VULNERABILITY.getDbConstant()),
qualityProfiles.stream()
.map(QualityProfile::getQpKey)
.collect(toList()));
.map(QualityProfile::getQpKey)
.collect(toList()));

Multimap<String, OrgActiveRuleDto> activeRulesByCategory = ArrayListMultimap.create();
activeRuleDtos
.forEach(r -> {
List<String> cwe = getCwe(r.getSecurityStandards());
if (includeCwe) {
cwe.forEach(s -> activeRulesByCategory.put(s, r));
}
switch (standard) {
case PARAM_OWASP_TOP_10:
getOwaspTop10(r.getSecurityStandards()).forEach(s -> activeRulesByCategory.put(s, r));
break;
case PARAM_SANS_TOP_25:
getSansTop25(cwe).forEach(s -> activeRulesByCategory.put(s, r));
break;
default:
throw new IllegalArgumentException("Unsupported standard: '" + standard + "'");
}
});
List<String> cwe = getCwe(r.getSecurityStandards());
if (includeCwe) {
cwe.forEach(s -> activeRulesByCategory.put(s, r));
}
switch (standard) {
case PARAM_OWASP_TOP_10:
getOwaspTop10(r.getSecurityStandards()).forEach(s -> activeRulesByCategory.put(s, r));
break;
case PARAM_SANS_TOP_25:
getSansTop25(cwe).forEach(s -> activeRulesByCategory.put(s, r));
break;
case PARAM_SONARSOURCE_SECURITY:
SecurityStandardHelper.getSonarSourceSecurityCategories(cwe).forEach(s -> activeRulesByCategory.put(s, r));
break;
default:
throw new IllegalArgumentException(String.format(UNSUPPORTED_STANDARD_MSG, standard));
}
});

List<RuleDto> ruleDtos = dbClient.ruleDao().selectByTypeAndLanguages(dbSession,
project.getOrganizationUuid(),
@@ -200,19 +212,22 @@ public class ShowAction implements SecurityReportsWsAction {
case PARAM_SANS_TOP_25:
getSansTop25(cwe).forEach(s -> rulesByCategory.put(s, r));
break;
case PARAM_SONARSOURCE_SECURITY:
getSonarSourceSecurityCategories(cwe).forEach(s -> rulesByCategory.put(s, r));
break;
default:
throw new IllegalArgumentException("Unsupported standard: '" + standard + "'");
throw new IllegalArgumentException(String.format(UNSUPPORTED_STANDARD_MSG, standard));
}
});

input.forEach(c -> {
c.setTotalRules(rulesByCategory.get(c.getCategory()).size());
c.setActiveRules(activeRulesByCategory.get(c.getCategory()).size());
c.getChildren().forEach(child -> {
child.setTotalRules(rulesByCategory.get(child.getCategory()).size());
child.setActiveRules(activeRulesByCategory.get(child.getCategory()).size());
});
c.setTotalRules(rulesByCategory.get(c.getCategory()).size());
c.setActiveRules(activeRulesByCategory.get(c.getCategory()).size());
c.getChildren().forEach(child -> {
child.setTotalRules(rulesByCategory.get(child.getCategory()).size());
child.setActiveRules(activeRulesByCategory.get(child.getCategory()).size());
});
});
}
}


+ 0
- 64
server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java Целия файл

@@ -23,7 +23,6 @@ import com.google.common.collect.ImmutableMap;
import java.util.Map;
import java.util.function.Consumer;
import javax.annotation.CheckForNull;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.junit.Rule;
@@ -39,7 +38,6 @@ import org.sonar.server.es.newindex.NewRegularIndex;
import org.sonar.server.es.newindex.SettingsConfiguration;

import static org.assertj.core.api.Assertions.assertThat;
import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
import static org.sonar.server.es.IndexType.main;
import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder;

@@ -99,60 +97,10 @@ public class IndexCreatorTest {
assertThat(metadataIndex.getInitialized(main(fakersIndex, "faker"))).isFalse();
}

@Test
public void recreate_index_on_definition_changes() {
// v1
startNewCreator(new FakeIndexDefinition());

IndexMainType fakeIndexType = main(Index.simple("fakes"), "fake");
String id = "1";
es.client().prepareIndex(fakeIndexType).setId(id).setSource(new FakeDoc().getFields()).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get();
assertThat(es.client().prepareGet(fakeIndexType, id).get().isExists()).isTrue();

// v2
startNewCreator(new FakeIndexDefinitionV2());

ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = mappings();
MappingMetaData mapping = mappings.get("fakes").get("fake");
assertThat(countMappingFields(mapping)).isEqualTo(3);
assertThat(field(mapping, "updatedAt").get("type")).isEqualTo("date");
assertThat(field(mapping, "newField").get("type")).isEqualTo("integer");

assertThat(es.client().prepareGet(fakeIndexType, id).get().isExists()).isFalse();
}

@Test
public void fail_to_recreate_index_on_definition_changes_if_blue_green_deployment() {
enableBlueGreenDeployment();

// v1
startNewCreator(new FakeIndexDefinition());

// v2
expectedException.expect(IllegalStateException.class);
expectedException.expectMessage("Blue/green deployment is not supported. Elasticsearch index [fakes] changed and needs to be dropped.");

startNewCreator(new FakeIndexDefinitionV2());
}

private void enableBlueGreenDeployment() {
settings.setProperty("sonar.blueGreenEnabled", "true");
}

@Test
public void do_not_recreate_index_on_unchanged_definition() {
// v1
startNewCreator(new FakeIndexDefinition());
IndexMainType fakeIndexType = main(Index.simple("fakes"), "fake");
String id = "1";
es.client().prepareIndex(fakeIndexType).setId(id).setSource(new FakeDoc().getFields()).setRefreshPolicy(IMMEDIATE).get();
assertThat(es.client().prepareGet(fakeIndexType, id).get().isExists()).isTrue();

// v1
startNewCreator(new FakeIndexDefinition());
assertThat(es.client().prepareGet(fakeIndexType, id).get().isExists()).isTrue();
}

@Test
public void delete_existing_indices_if_db_vendor_changed() {
testDeleteOnDbChange(LOG_DB_VENDOR_CHANGED,
@@ -244,16 +192,4 @@ public class IndexCreatorTest {
.createDateTimeField("updatedAt");
}
}
private static class FakeIndexDefinitionV2 implements IndexDefinition {

@Override
public void define(IndexDefinitionContext context) {
Index index = Index.simple("fakes");
NewRegularIndex newIndex = context.create(index, SETTINGS_CONFIGURATION);
newIndex.createTypeMapping(IndexType.main(index, "fake"))
.keywordFieldBuilder("key").build()
.createDateTimeField("updatedAt")
.createIntegerField("newField");
}
}
}

+ 34
- 0
server/sonar-server/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java Целия файл

@@ -20,6 +20,7 @@
package org.sonar.server.es;

import java.util.Iterator;
import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
import org.junit.Rule;
import org.junit.Test;
import org.sonar.api.config.internal.MapSettings;
@@ -66,6 +67,39 @@ public class MigrationEsClientImplTest {
.doesNotContain("Drop Elasticsearch index [xxx]");
}

@Test
public void add_mapping_to_existing_index() {
underTest.addMappingToExistingIndex("as", "s", "newMapping", "keyword");

GetFieldMappingsResponse response = es.client().nativeClient().admin().indices().prepareGetFieldMappings("as")
.setTypes("s")
.setFields("newMapping")
.get();
assertThat(response).isNotNull();
assertThat(response.mappings()).hasSize(1);
assertThat(response.mappings().get("as")).isNotNull();
assertThat(response.mappings().get("as").get("s")).isNotNull();
assertThat(response.mappings().get("as").get("s").get("newMapping").fullName()).isEqualTo("newMapping");
assertThat(logTester.logs(LoggerLevel.INFO))
.contains("Add mapping [newMapping] to Elasticsearch index [as]");
}

@Test
public void add_mapping_to_non_existing_index() {
underTest.addMappingToExistingIndex("yyyy", "s", "newMapping", "keyword");

GetFieldMappingsResponse response = es.client().nativeClient().admin().indices().prepareGetFieldMappings("as")
.setTypes("s")
.setFields("newMapping")
.get();
assertThat(response).isNotNull();
assertThat(response.mappings()).hasSize(1);
assertThat(response.mappings().get("as")).isNotNull();
assertThat(response.mappings().get("as").get("s")).isNotNull();
assertThat(response.mappings().get("as").get("s").get("newMapping").fullName()).isEqualTo("");
assertThat(logTester.logs(LoggerLevel.INFO)).isEmpty();
}

private Iterator<String> loadExistingIndices() {
return es.client().nativeClient().admin().indices().prepareGetMappings().get().mappings().keysIt();
}

+ 72
- 0
server/sonar-server/src/test/java/org/sonar/server/securityreport/ws/ShowActionTest.java Целия файл

@@ -306,6 +306,78 @@ public class ShowActionTest {
.isSimilarTo(this.getClass().getResource("ShowActionTest/sansWithCwe.json"));
}

@Test
public void sonarsource_security_without_cwe() {
userSessionRule.addProjectPermission(UserRole.USER, project);
indexPermissions();
ComponentDto file = insertComponent(newFileDto(project, null, "FILE_ID").setDbKey("FILE_KEY"));
IssueDto issue1 = newIssue(rule1, project, file)
.setStatus("OPEN")
.setSeverity("MAJOR")
.setType(RuleType.VULNERABILITY);
IssueDto issue2 = newIssue(rule1, project, file)
.setStatus("OPEN")
.setSeverity("MAJOR")
.setType(RuleType.SECURITY_HOTSPOT);
IssueDto issue3 = newIssue(rule1, project, file)
.setStatus(Issue.STATUS_RESOLVED)
.setResolution(Issue.RESOLUTION_FIXED)
.setSeverity("MAJOR")
.setType(RuleType.SECURITY_HOTSPOT);
IssueDto issue4 = newIssue(rule1, project, file)
.setStatus(Issue.STATUS_RESOLVED)
.setResolution(Issue.RESOLUTION_WONT_FIX)
.setSeverity("MAJOR")
.setType(RuleType.SECURITY_HOTSPOT);
dbClient.issueDao().insert(session, issue1, issue2, issue3, issue4);
session.commit();
indexIssues();

assertJson(ws.newRequest()
.setParam("standard", "sonarsourceSecurity")
.setParam("project", project.getKey())
.setParam("includeDistribution", "false")
.execute().getInput())
.withStrictArrayOrder()
.isSimilarTo(this.getClass().getResource("ShowActionTest/sonarsourceSecurityNoCwe.json"));
}

@Test
public void sonarsource_security_with_cwe() {
userSessionRule.addProjectPermission(UserRole.USER, project);
indexPermissions();
ComponentDto file = insertComponent(newFileDto(project, null, "FILE_ID").setDbKey("FILE_KEY"));
IssueDto issue1 = newIssue(rule1, project, file)
.setStatus("OPEN")
.setSeverity("MAJOR")
.setType(RuleType.VULNERABILITY);
IssueDto issue2 = newIssue(rule1, project, file)
.setStatus("OPEN")
.setSeverity("MAJOR")
.setType(RuleType.SECURITY_HOTSPOT);
IssueDto issue3 = newIssue(rule1, project, file)
.setStatus(Issue.STATUS_RESOLVED)
.setResolution(Issue.RESOLUTION_FIXED)
.setSeverity("MAJOR")
.setType(RuleType.SECURITY_HOTSPOT);
IssueDto issue4 = newIssue(rule1, project, file)
.setStatus(Issue.STATUS_RESOLVED)
.setResolution(Issue.RESOLUTION_WONT_FIX)
.setSeverity("MAJOR")
.setType(RuleType.SECURITY_HOTSPOT);
dbClient.issueDao().insert(session, issue1, issue2, issue3, issue4);
session.commit();
indexIssues();

assertJson(ws.newRequest()
.setParam("standard", "sonarsourceSecurity")
.setParam("project", project.getKey())
.setParam("includeDistribution", "true")
.execute().getInput())
.withStrictArrayOrder()
.isSimilarTo(this.getClass().getResource("ShowActionTest/sonarsourceSecurityWithCwe.json"));
}

private RuleDefinitionDto newRule(Set tags) {
RuleDefinitionDto rule = RuleTesting.newRule()
.setType(RuleType.SECURITY_HOTSPOT)

+ 205
- 0
server/sonar-server/src/test/resources/org/sonar/server/securityreport/ws/ShowActionTest/sonarsourceSecurityNoCwe.json Целия файл

@@ -0,0 +1,205 @@
{
"categories": [
{
"category": "ldap-injection",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "object-injection",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "ssrf",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "insecure-conf",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "xxe",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "auth",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "xpath-injection",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "weak-cryptography",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "dos",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "open-redirect",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "log-injection",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "csrf",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "sql-injection",
"vulnerabilities": 1,
"vulnerabilityRating": 3,
"toReviewSecurityHotspots": 1,
"openSecurityHotspots": 1,
"wontFixSecurityHotspots": 1,
"distribution": [],
"activeRules": 1,
"totalRules": 1
},
{
"category": "file-manipulation",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "expression-lang-injection",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "rce",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "xss",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "path-traversal-injection",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 1,
"totalRules": 1
},
{
"category": "command-injection",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "http-response-splitting",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
}
]
}

+ 226
- 0
server/sonar-server/src/test/resources/org/sonar/server/securityreport/ws/ShowActionTest/sonarsourceSecurityWithCwe.json Целия файл

@@ -0,0 +1,226 @@
{
"categories": [
{
"category": "ldap-injection",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "object-injection",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "ssrf",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "insecure-conf",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "xxe",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "auth",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "xpath-injection",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "weak-cryptography",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "dos",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "open-redirect",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "log-injection",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "csrf",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "sql-injection",
"vulnerabilities": 1,
"vulnerabilityRating": 3,
"toReviewSecurityHotspots": 1,
"openSecurityHotspots": 1,
"wontFixSecurityHotspots": 1,
"distribution": [
{
"cwe": "89",
"vulnerabilities": 1,
"vulnerabilityRating": 3,
"toReviewSecurityHotspots": 1,
"openSecurityHotspots": 1,
"wontFixSecurityHotspots": 1,
"activeRules": 1,
"totalRules": 1
},
{
"cwe": "123",
"vulnerabilities": 1,
"vulnerabilityRating": 3,
"toReviewSecurityHotspots": 1,
"openSecurityHotspots": 1,
"wontFixSecurityHotspots": 1,
"activeRules": 1,
"totalRules": 1
}
],
"activeRules": 1,
"totalRules": 1
},
{
"category": "file-manipulation",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "expression-lang-injection",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "rce",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "xss",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "path-traversal-injection",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 1,
"totalRules": 1
},
{
"category": "command-injection",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
},
{
"category": "http-response-splitting",
"vulnerabilities": 0,
"toReviewSecurityHotspots": 0,
"openSecurityHotspots": 0,
"wontFixSecurityHotspots": 0,
"distribution": [],
"activeRules": 0,
"totalRules": 0
}
]
}

+ 1
- 0
sonar-ws/src/main/java/org/sonarqube/ws/client/issue/IssuesWsParameters.java Целия файл

@@ -85,6 +85,7 @@ public class IssuesWsParameters {
public static final String PARAM_TYPES = "types";
public static final String PARAM_OWASP_TOP_10 = "owaspTop10";
public static final String PARAM_SANS_TOP_25 = "sansTop25";
public static final String PARAM_SONARSOURCE_SECURITY = "sonarsourceSecurity";
public static final String PARAM_CWE = "cwe";
public static final String PARAM_ASSIGNED = "assigned";
public static final String PARAM_HIDE_COMMENTS = "hideComments";

Loading…
Отказ
Запис