diff options
author | Belen Pruvost <belen.pruvost@sonarsource.com> | 2022-01-18 17:50:47 +0100 |
---|---|---|
committer | sonartech <sonartech@sonarsource.com> | 2022-01-21 20:03:22 +0000 |
commit | 9d1361c43487f91b3d001a7e1385d35fa05a5115 (patch) | |
tree | be3c11961e8854ff7afc0a9c85555b9fcb8c8412 | |
parent | c27e6f711185a6b6b64e381f9c8b152b4ac999e8 (diff) | |
download | sonarqube-9d1361c43487f91b3d001a7e1385d35fa05a5115.tar.gz sonarqube-9d1361c43487f91b3d001a7e1385d35fa05a5115.zip |
SONAR-15904 - Adapt Issue Search to work with new code reference
11 files changed, 312 insertions, 40 deletions
diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueDoc.java b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueDoc.java index adfb6c01d4f..a567526d86c 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueDoc.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueDoc.java @@ -336,4 +336,13 @@ public class IssueDoc extends BaseDoc { setField(IssueIndexDefinition.FIELD_ISSUE_VULNERABILITY_PROBABILITY, v == null ? null : v.getScore()); return this; } + + public boolean isNewCodeReference() { + return getField(IssueIndexDefinition.FIELD_ISSUE_NEW_CODE_REFERENCE); + } + + public IssueDoc setIsNewCodeReference(boolean b) { + setField(IssueIndexDefinition.FIELD_ISSUE_NEW_CODE_REFERENCE, b); + return this; + } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java index 210174a02b3..08aa67ad344 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java @@ -102,6 +102,11 @@ public class IssueIndexDefinition implements IndexDefinition { public static final String FIELD_ISSUE_SQ_SECURITY_CATEGORY = "sonarsourceSecurity"; public static final String FIELD_ISSUE_VULNERABILITY_PROBABILITY = "vulnerabilityProbability"; + /** + * Whether issue is new code for a branch using the reference branch new code definition. + */ + public static final String FIELD_ISSUE_NEW_CODE_REFERENCE = "isNewCodeReference"; + private final Configuration config; private final boolean enableSource; @@ -163,5 +168,6 @@ public class IssueIndexDefinition implements IndexDefinition { mapping.keywordFieldBuilder(FIELD_ISSUE_CWE).disableNorms().build(); mapping.keywordFieldBuilder(FIELD_ISSUE_SQ_SECURITY_CATEGORY).disableNorms().build(); mapping.keywordFieldBuilder(FIELD_ISSUE_VULNERABILITY_PROBABILITY).disableNorms().build(); + mapping.createBooleanField(FIELD_ISSUE_NEW_CODE_REFERENCE); } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIteratorForSingleChunk.java b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIteratorForSingleChunk.java index c07b830cabf..7aba28607de 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIteratorForSingleChunk.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIteratorForSingleChunk.java @@ -42,6 +42,7 @@ import org.sonar.db.ResultSetIterator; import org.sonar.server.security.SecurityStandards; import static com.google.common.base.Preconditions.checkArgument; +import static org.elasticsearch.common.Strings.isNullOrEmpty; import static org.sonar.api.utils.DateUtils.longToDate; import static org.sonar.db.DatabaseUtils.getLong; import static org.sonar.db.rule.RuleDefinitionDto.deserializeSecurityStandardsString; @@ -81,16 +82,19 @@ class IssueIteratorForSingleChunk implements IssueIterator { "i.tags", "i.issue_type", "r.security_standards", - "c.qualifier" + "c.qualifier", + "n.uuid" }; private static final String SQL_ALL = "select " + StringUtils.join(FIELDS, ",") + " from issues i " + "inner join rules r on r.uuid = i.rule_uuid " + "inner join components c on c.uuid = i.component_uuid "; + private static final String SQL_NEW_CODE_JOIN = "left join new_code_reference_issues n on n.issue_key = i.kee "; + private static final String PROJECT_FILTER = " and c.project_uuid = ? and i.project_uuid = ? "; private static final String ISSUE_KEY_FILTER_PREFIX = " and i.kee in ("; - private static final String ISSUE_KEY_FILTER_SUFFIX = ")"; + private static final String ISSUE_KEY_FILTER_SUFFIX = ") "; static final Splitter TAGS_SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings(); static final Splitter MODULE_PATH_SPLITTER = Splitter.on('.').trimResults().omitEmptyStrings(); @@ -151,6 +155,7 @@ class IssueIteratorForSingleChunk implements IssueIterator { sql += IntStream.range(0, issueKeys.size()).mapToObj(i -> "?").collect(Collectors.joining(",")); sql += ISSUE_KEY_FILTER_SUFFIX; } + sql += SQL_NEW_CODE_JOIN; return sql; } @@ -237,6 +242,7 @@ class IssueIteratorForSingleChunk implements IssueIterator { doc.setVulnerabilityProbability(sqCategory.getVulnerability()); doc.setScope(Qualifiers.UNIT_TEST_FILE.equals(rs.getString(23)) ? IssueScope.TEST : IssueScope.MAIN); + doc.setIsNewCodeReference(!isNullOrEmpty(rs.getString(24))); return doc; } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/issue/index/IssueIteratorFactoryTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/issue/index/IssueIteratorFactoryTest.java index 512c0db0cd4..ab5ac125482 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/issue/index/IssueIteratorFactoryTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/issue/index/IssueIteratorFactoryTest.java @@ -37,6 +37,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.sonar.db.component.ComponentTesting.newDirectory; import static org.sonar.db.component.ComponentTesting.newFileDto; import static org.sonar.db.component.ComponentTesting.newModuleDto; +import static org.sonar.db.issue.IssueTesting.newCodeReferenceIssue; public class IssueIteratorFactoryTest { @@ -119,11 +120,15 @@ public class IssueIteratorFactoryTest { IssueDto dirIssue = dbTester.issues().insert(rule, project, directory); IssueDto projectIssue = dbTester.issues().insert(rule, project, project); + dbTester.issues().insertNewCodeReferenceIssue(newCodeReferenceIssue(fileIssue)); + Map<String, IssueDoc> issuesByKey = issuesByKey(); assertThat(issuesByKey) .hasSize(4) .containsOnlyKeys(fileIssue.getKey(), moduleIssue.getKey(), dirIssue.getKey(), projectIssue.getKey()); + + assertThat(issuesByKey.get(fileIssue.getKey()).isNewCodeReference()).isTrue(); } @Test diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java index 09313ee6909..5b44483753a 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java @@ -149,6 +149,7 @@ import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_LANG import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_LINE; import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_MODULE_PATH; import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_MODULE_UUID; +import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_NEW_CODE_REFERENCE; import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_OWASP_TOP_10; import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID; import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_RESOLUTION; @@ -452,6 +453,8 @@ public class IssueIndex { addComponentRelatedFilters(query, filters); addDatesFilter(filters, query); addCreatedAfterByProjectsFilter(filters, query); + addNewCodeReferenceFilter(filters, query); + addNewCodeReferenceFilterByProjectsFilter(filters, query); return filters; } @@ -562,11 +565,11 @@ public class IssueIndex { private static RequestFiltersComputer newFilterComputer(SearchOptions options, AllFilters allFilters) { Collection<String> facetNames = options.getFacets(); Set<TopAggregationDefinition<?>> facets = Stream.concat( - Stream.of(EFFORT_TOP_AGGREGATION), - facetNames.stream() - .map(FACETS_BY_NAME::get) - .filter(Objects::nonNull) - .map(Facet::getTopAggregationDef)) + Stream.of(EFFORT_TOP_AGGREGATION), + facetNames.stream() + .map(FACETS_BY_NAME::get) + .filter(Objects::nonNull) + .map(Facet::getTopAggregationDef)) .collect(MoreCollectors.toSet(facetNames.size())); return new RequestFiltersComputer(allFilters, facets); @@ -645,6 +648,28 @@ public class IssueIndex { } } + private static void addNewCodeReferenceFilter(AllFilters filters, IssueQuery query) { + Boolean newCodeOnReference = query.newCodeOnReference(); + + if (newCodeOnReference != null) { + filters.addFilter( + FIELD_ISSUE_NEW_CODE_REFERENCE, new SimpleFieldFilterScope(FIELD_ISSUE_NEW_CODE_REFERENCE), + termQuery(FIELD_ISSUE_NEW_CODE_REFERENCE, true)); + } + } + + private static void addNewCodeReferenceFilterByProjectsFilter(AllFilters allFilters, IssueQuery query) { + Collection<String> newCodeOnReferenceByProjectUuids = query.newCodeOnReferenceByProjectUuids(); + BoolQueryBuilder boolQueryBuilder = boolQuery(); + + newCodeOnReferenceByProjectUuids.forEach(projectOrProjectBranchUuid -> boolQueryBuilder.should(boolQuery() + .filter(termQuery(FIELD_ISSUE_BRANCH_UUID, projectOrProjectBranchUuid)) + .filter(termQuery(FIELD_ISSUE_NEW_CODE_REFERENCE, true)))); + + allFilters.addFilter("__is_new_code_reference_by_project_uuids", new SimpleFieldFilterScope("newCodeReferenceByProjectUuids"), boolQueryBuilder); + } + + private static void addCreatedAfterByProjectsFilter(AllFilters allFilters, IssueQuery query) { Map<String, PeriodStart> createdAfterByProjectUuids = query.createdAfterByProjectUuids(); BoolQueryBuilder boolQueryBuilder = boolQuery(); @@ -865,7 +890,7 @@ public class IssueIndex { t -> // add sub-aggregation to return issue count for current user aggregationHelper.getSubAggregationHelper() - .buildSelectedItemsAggregation(ASSIGNED_TO_ME.getName(), ASSIGNED_TO_ME.getTopAggregationDef(), new String[] {uuid}) + .buildSelectedItemsAggregation(ASSIGNED_TO_ME.getName(), ASSIGNED_TO_ME.getTopAggregationDef(), new String[]{uuid}) .ifPresent(t::subAggregation)); esRequest.aggregation(aggregation); } diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueQuery.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueQuery.java index d3be5a82a4f..c18b04fa4a1 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueQuery.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueQuery.java @@ -97,6 +97,8 @@ public class IssueQuery { private final String branchUuid; private final Boolean mainBranch; private final ZoneId timeZone; + private final Boolean newCodeOnReference; + private final Collection<String> newCodeOnReferenceByProjectUuids; private IssueQuery(Builder builder) { this.issueKeys = defaultCollection(builder.issueKeys); @@ -135,6 +137,8 @@ public class IssueQuery { this.branchUuid = builder.branchUuid; this.mainBranch = builder.mainBranch; this.timeZone = builder.timeZone; + this.newCodeOnReference = builder.newCodeOnReference; + this.newCodeOnReferenceByProjectUuids = defaultCollection(builder.newCodeOnReferenceByProjectUuids); } public Collection<String> issueKeys() { @@ -300,6 +304,15 @@ public class IssueQuery { return timeZone; } + @CheckForNull + public Boolean newCodeOnReference() { + return newCodeOnReference; + } + + public Collection<String> newCodeOnReferenceByProjectUuids() { + return newCodeOnReferenceByProjectUuids; + } + public static class Builder { private Collection<String> issueKeys; private Collection<String> severities; @@ -337,6 +350,8 @@ public class IssueQuery { private String branchUuid; private Boolean mainBranch = true; private ZoneId timeZone; + private Boolean newCodeOnReference = null; + private Collection<String> newCodeOnReferenceByProjectUuids; private Builder() { @@ -548,6 +563,16 @@ public class IssueQuery { this.timeZone = timeZone; return this; } + + public Builder newCodeOnReference(@Nullable Boolean newCodeOnReference) { + this.newCodeOnReference = newCodeOnReference; + return this; + } + + public Builder newCodeOnReferenceByProjectUuids(@Nullable Collection<String> newCodeOnReferenceByProjectUuids) { + this.newCodeOnReferenceByProjectUuids = newCodeOnReferenceByProjectUuids; + return this; + } } private static <T> Collection<T> defaultCollection(@Nullable Collection<T> c) { diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueQueryFactory.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueQueryFactory.java index 5647bab8e1f..fd7a17885ca 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueQueryFactory.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueQueryFactory.java @@ -20,7 +20,6 @@ package org.sonar.server.issue.index; import com.google.common.base.Joiner; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import java.time.Clock; import java.time.DateTimeException; @@ -57,6 +56,7 @@ import org.sonar.server.issue.index.IssueQuery.PeriodStart; import org.sonar.server.user.UserSession; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Strings.isNullOrEmpty; import static com.google.common.collect.Collections2.transform; import static java.lang.String.format; import static java.util.Collections.singleton; @@ -72,6 +72,7 @@ import static org.sonar.core.util.stream.MoreCollectors.toHashSet; import static org.sonar.core.util.stream.MoreCollectors.toList; import static org.sonar.core.util.stream.MoreCollectors.toSet; import static org.sonar.core.util.stream.MoreCollectors.uniqueIndex; +import static org.sonar.db.newcodeperiod.NewCodePeriodType.REFERENCE_BRANCH; import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_COMPONENT_KEYS; import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_COMPONENT_UUIDS; import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_CREATED_AFTER; @@ -146,7 +147,7 @@ public class IssueQueryFactory { setCreatedAfterFromRequest(dbSession, builder, request, allComponents, timeZone); String sort = request.getSort(); - if (!Strings.isNullOrEmpty(sort)) { + if (!isNullOrEmpty(sort)) { builder.sort(sort); builder.asc(request.getAsc()); } @@ -184,6 +185,7 @@ public class IssueQueryFactory { checkArgument(createdAfter == null || createdInLast == null, format("Parameters %s and %s cannot be set simultaneously", PARAM_CREATED_AFTER, PARAM_CREATED_IN_LAST)); setCreatedAfterFromDates(builder, createdAfter, createdInLast, true); } else { + // If the filter is on leak period checkArgument(createdAfter == null, "Parameters '%s' and '%s' cannot be set simultaneously", PARAM_CREATED_AFTER, PARAM_SINCE_LEAK_PERIOD); checkArgument(createdInLast == null, format("Parameters %s and %s cannot be set simultaneously", PARAM_CREATED_IN_LAST, PARAM_SINCE_LEAK_PERIOD)); @@ -191,18 +193,36 @@ public class IssueQueryFactory { ComponentDto component = componentUuids.iterator().next(); if (!QUALIFIERS_WITHOUT_LEAK_PERIOD.contains(component.qualifier()) && request.getPullRequest() == null) { - Date createdAfterFromSnapshot = findCreatedAfterFromComponentUuid(dbSession, component); - setCreatedAfterFromDates(builder, createdAfterFromSnapshot, null, false); + Optional<SnapshotDto> snapshot = getLastAnalysis(dbSession, component); + boolean isLastAnalysisUsingReferenceBranch = isLastAnalysisUsingReferenceBranch(snapshot); + if (isLastAnalysisUsingReferenceBranch) { + builder.newCodeOnReference(true); + } else { + // if last analysis has no period date, then no issue should be considered new. + Date createdAfterFromSnapshot = findCreatedAfterFromComponentUuid(snapshot); + setCreatedAfterFromDates(builder, createdAfterFromSnapshot, null, false); + } } } } - private Date findCreatedAfterFromComponentUuid(DbSession dbSession, ComponentDto component) { - Optional<SnapshotDto> snapshot = dbClient.snapshotDao().selectLastAnalysisByComponentUuid(dbSession, component.uuid()); - // if last analysis has no period date, then no issue should be considered new. + private Date findCreatedAfterFromComponentUuid(Optional<SnapshotDto> snapshot) { return snapshot.map(s -> longToDate(s.getPeriodDate())).orElseGet(() -> new Date(clock.millis())); } + private static boolean isLastAnalysisUsingReferenceBranch(Optional<SnapshotDto> snapshot) { + String periodMode = snapshot.map(SnapshotDto::getPeriodMode).orElse(""); + return periodMode.equals(REFERENCE_BRANCH.name()); + } + + private Optional<SnapshotDto> getLastAnalysis(DbSession dbSession, ComponentDto component) { + return dbClient.snapshotDao().selectLastAnalysisByComponentUuid(dbSession, component.uuid()); + } + + private List<SnapshotDto> getLastAnalysis(DbSession dbSession, Set<String> projectUuids) { + return dbClient.snapshotDao().selectLastAnalysesByRootComponentUuids(dbSession, projectUuids); + } + private boolean mergeDeprecatedComponentParameters(DbSession session, SearchRequest request, List<ComponentDto> allComponents) { Boolean onComponentOnly = request.getOnComponentOnly(); Collection<String> components = request.getComponents(); @@ -332,12 +352,22 @@ public class IssueQueryFactory { .flatMap(app -> dbClient.componentDao().selectProjectsFromView(dbSession, app, app).stream()) .collect(toSet()); - Map<String, PeriodStart> leakByProjects = dbClient.snapshotDao().selectLastAnalysesByRootComponentUuids(dbSession, projectUuids) + List<SnapshotDto> snapshots = getLastAnalysis(dbSession, projectUuids); + + Set<String> newCodeReferenceByProjects = snapshots + .stream() + .filter(s -> !isNullOrEmpty(s.getPeriodMode()) && s.getPeriodMode().equals(REFERENCE_BRANCH.name())) + .map(SnapshotDto::getComponentUuid) + .collect(toSet()); + + Map<String, PeriodStart> leakByProjects = snapshots .stream() - .filter(s -> s.getPeriodDate() != null) + .filter(s -> s.getPeriodDate() != null && + (isNullOrEmpty(s.getPeriodMode()) || !s.getPeriodMode().equals(REFERENCE_BRANCH.name()))) .collect(uniqueIndex(SnapshotDto::getComponentUuid, s -> new PeriodStart(longToDate(s.getPeriodDate()), false))); builder.createdAfterByProjectUuids(leakByProjects); + builder.newCodeOnReferenceByProjectUuids(newCodeReferenceByProjects); } private static void addDirectories(IssueQuery.Builder builder, List<ComponentDto> directories) { diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexFiltersTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexFiltersTest.java index d03c3b9fe3e..ab02c967806 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexFiltersTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexFiltersTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Arrays; import java.util.Date; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import org.assertj.core.api.Fail; import org.elasticsearch.search.SearchHit; @@ -502,6 +503,51 @@ public class IssueIndexFiltersTest { } @Test + public void filter_by_new_code_reference_by_projects() { + ComponentDto project1 = newPrivateProjectDto(); + IssueDoc project1Issue1 = newDoc(project1).setIsNewCodeReference(true); + IssueDoc project1Issue2 = newDoc(project1).setIsNewCodeReference(false); + ComponentDto project2 = newPrivateProjectDto(); + IssueDoc project2Issue1 = newDoc(project2).setIsNewCodeReference(false); + IssueDoc project2Issue2 = newDoc(project2).setIsNewCodeReference(true); + indexIssues(project1Issue1, project1Issue2, project2Issue1, project2Issue2); + + // Search for issues of project 1 and project 2 that are new code on a branch using reference for new code + assertThatSearchReturnsOnly(IssueQuery.builder() + .newCodeOnReferenceByProjectUuids(Set.of(project1.uuid(), project2.uuid())), + project1Issue1.key(), project2Issue2.key()); + } + + @Test + public void filter_by_new_code_reference_branches() { + ComponentDto project1 = newPrivateProjectDto(); + IssueDoc project1Issue1 = newDoc(project1).setIsNewCodeReference(true); + IssueDoc project1Issue2 = newDoc(project1).setIsNewCodeReference(false); + + ComponentDto project1Branch1 = db.components().insertProjectBranch(project1); + IssueDoc project1Branch1Issue1 = newDoc(project1Branch1).setIsNewCodeReference(false); + IssueDoc project1Branch1Issue2 = newDoc(project1Branch1).setIsNewCodeReference(true); + + ComponentDto project2 = newPrivateProjectDto(); + + IssueDoc project2Issue1 = newDoc(project2).setIsNewCodeReference(true); + IssueDoc project2Issue2 = newDoc(project2).setIsNewCodeReference(false); + + ComponentDto project2Branch1 = db.components().insertProjectBranch(project2); + IssueDoc project2Branch1Issue1 = newDoc(project2Branch1).setIsNewCodeReference(false); + IssueDoc project2Branch1Issue2 = newDoc(project2Branch1).setIsNewCodeReference(true); + + indexIssues(project1Issue1, project1Issue2, project2Issue1, project2Issue2, + project1Branch1Issue1, project1Branch1Issue2, project2Branch1Issue1, project2Branch1Issue2); + + // Search for issues of project 1 branch 1 and project 2 branch 1 that are new code on a branch using reference for new code + assertThatSearchReturnsOnly(IssueQuery.builder() + .mainBranch(false) + .newCodeOnReferenceByProjectUuids(Set.of(project1Branch1.uuid(), project2Branch1.uuid())), + project1Branch1Issue2.key(), project2Branch1Issue2.key()); + } + + @Test public void filter_by_severities() { ComponentDto project = newPrivateProjectDto(); ComponentDto file = newFileDto(project, null); @@ -757,6 +803,17 @@ public class IssueIndexFiltersTest { } @Test + public void filter_by_new_code_reference() { + ComponentDto project = newPrivateProjectDto(); + ComponentDto file = newFileDto(project, null); + + indexIssues(newDoc("I1", file).setIsNewCodeReference(true), + newDoc("I2", file).setIsNewCodeReference(false)); + + assertThatSearchReturnsOnly(IssueQuery.builder().newCodeOnReference(true), "I1"); + } + + @Test public void filter_by_cwe() { ComponentDto project = newPrivateProjectDto(); ComponentDto file = newFileDto(project, null); diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueQueryFactoryTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueQueryFactoryTest.java index a04621b98dd..1de6246a60e 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueQueryFactoryTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueQueryFactoryTest.java @@ -56,6 +56,7 @@ import static org.sonar.db.component.ComponentTesting.newFileDto; import static org.sonar.db.component.ComponentTesting.newModuleDto; import static org.sonar.db.component.ComponentTesting.newProjectCopy; import static org.sonar.db.component.ComponentTesting.newSubPortfolio; +import static org.sonar.db.newcodeperiod.NewCodePeriodType.REFERENCE_BRANCH; import static org.sonar.db.rule.RuleTesting.newRule; public class IssueQueryFactoryTest { @@ -163,7 +164,29 @@ public class IssueQueryFactoryTest { assertThat(query.componentUuids()).containsOnly(file.uuid()); assertThat(query.createdAfter().date()).isEqualTo(new Date(leakPeriodStart)); assertThat(query.createdAfter().inclusive()).isFalse(); + assertThat(query.newCodeOnReference()).isNull(); + } + + @Test + public void leak_period_does_not_rely_on_date_for_reference_branch() { + long leakPeriodStart = addDays(new Date(), -14).getTime(); + + ComponentDto project = db.components().insertPublicProject(); + ComponentDto file = db.components().insertComponent(newFileDto(project)); + + SnapshotDto analysis = db.components().insertSnapshot(project, s -> s.setPeriodMode(REFERENCE_BRANCH.name()) + .setPeriodParam("master")); + SearchRequest request = new SearchRequest() + .setComponentUuids(Collections.singletonList(file.uuid())) + .setOnComponentOnly(true) + .setSinceLeakPeriod(true); + + IssueQuery query = underTest.create(request); + + assertThat(query.componentUuids()).containsOnly(file.uuid()); + assertThat(query.newCodeOnReference()).isTrue(); + assertThat(query.createdAfter()).isNull(); } @Test @@ -319,11 +342,15 @@ public class IssueQueryFactoryTest { ComponentDto project2 = db.components().insertPublicProject(); db.components().insertSnapshot(project2, s -> s.setPeriodDate(null)); ComponentDto project3 = db.components().insertPublicProject(); + ComponentDto project4 = db.components().insertPublicProject(); + SnapshotDto analysis2 = db.components().insertSnapshot(project4, + s -> s.setPeriodMode(REFERENCE_BRANCH.name()).setPeriodParam("master")); ComponentDto application = db.components().insertPublicApplication(); db.components().insertComponents(newProjectCopy("PC1", project1, application)); db.components().insertComponents(newProjectCopy("PC2", project2, application)); db.components().insertComponents(newProjectCopy("PC3", project3, application)); - userSession.registerApplication(application, project1, project2, project3); + db.components().insertComponents(newProjectCopy("PC4", project4, application)); + userSession.registerApplication(application, project1, project2, project3, project4); IssueQuery result = underTest.create(new SearchRequest() .setComponentUuids(singletonList(application.uuid())) @@ -332,6 +359,8 @@ public class IssueQueryFactoryTest { assertThat(result.createdAfterByProjectUuids()).hasSize(1); assertThat(result.createdAfterByProjectUuids().entrySet()).extracting(Map.Entry::getKey, e -> e.getValue().date(), e -> e.getValue().inclusive()).containsOnly( tuple(project1.uuid(), new Date(analysis1.getPeriodDate()), false)); + assertThat(result.newCodeOnReferenceByProjectUuids()).hasSize(1); + assertThat(result.newCodeOnReferenceByProjectUuids()).containsOnly(project4.uuid()); assertThat(result.viewUuids()).containsExactlyInAnyOrder(application.uuid()); } diff --git a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/hotspot/ws/SearchAction.java b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/hotspot/ws/SearchAction.java index 076bab9027e..53c9038327b 100644 --- a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/hotspot/ws/SearchAction.java +++ b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/hotspot/ws/SearchAction.java @@ -19,7 +19,6 @@ */ package org.sonar.server.hotspot.ws; -import com.google.common.collect.ImmutableSet; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -37,6 +36,7 @@ import javax.annotation.Nullable; import org.apache.lucene.search.TotalHits; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.search.SearchHit; +import org.jetbrains.annotations.NotNull; import org.sonar.api.resources.Qualifiers; import org.sonar.api.resources.Scopes; import org.sonar.api.rule.RuleKey; @@ -67,6 +67,7 @@ import org.sonarqube.ws.Hotspots.SearchWsResponse; import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Strings.isNullOrEmpty; import static java.lang.String.format; import static java.util.Collections.singleton; import static java.util.Collections.singletonList; @@ -82,7 +83,9 @@ import static org.sonar.api.utils.DateUtils.longToDate; import static org.sonar.api.utils.Paging.forPageIndex; import static org.sonar.api.web.UserRole.USER; import static org.sonar.core.util.stream.MoreCollectors.toList; +import static org.sonar.core.util.stream.MoreCollectors.toSet; import static org.sonar.core.util.stream.MoreCollectors.uniqueIndex; +import static org.sonar.db.newcodeperiod.NewCodePeriodType.REFERENCE_BRANCH; import static org.sonar.server.security.SecurityStandards.SANS_TOP_25_INSECURE_INTERACTION; import static org.sonar.server.security.SecurityStandards.SANS_TOP_25_POROUS_DEFENSES; import static org.sonar.server.security.SecurityStandards.SANS_TOP_25_RISKY_RESOURCE; @@ -94,7 +97,7 @@ import static org.sonar.server.ws.WsUtils.writeProtobuf; import static org.sonarqube.ws.WsUtils.nullToEmpty; public class SearchAction implements HotspotsWsAction { - private static final Set<String> SUPPORTED_QUALIFIERS = ImmutableSet.of(Qualifiers.PROJECT, Qualifiers.APP); + private static final Set<String> SUPPORTED_QUALIFIERS = Set.of(Qualifiers.PROJECT, Qualifiers.APP); private static final String PARAM_PROJECT_KEY = "projectKey"; private static final String PARAM_STATUS = "status"; private static final String PARAM_RESOLUTION = "resolution"; @@ -130,7 +133,7 @@ public class SearchAction implements HotspotsWsAction { } private static Set<String> setFromList(@Nullable List<String> list) { - return list != null ? ImmutableSet.copyOf(list) : ImmutableSet.of(); + return list != null ? Set.copyOf(list) : Set.of(); } private static WsRequest toWsRequest(Request request) { @@ -344,24 +347,16 @@ public class SearchAction implements HotspotsWsAction { if (Qualifiers.APP.equals(project.qualifier())) { builder.viewUuids(singletonList(projectUuid)); if (wsRequest.isSinceLeakPeriod() && wsRequest.getPullRequest().isEmpty()) { - addCreatedAfterByProjects(builder, dbSession, project); + addSinceLeakPeriodFilterByProjects(builder, dbSession, project); } } else { builder.projectUuids(singletonList(projectUuid)); if (wsRequest.isSinceLeakPeriod() && wsRequest.getPullRequest().isEmpty()) { - var sinceDate = dbClient.snapshotDao().selectLastAnalysisByComponentUuid(dbSession, project.uuid()) - .map(s -> longToDate(s.getPeriodDate())) - .orElseGet(() -> new Date(system2.now())); - builder.createdAfter(sinceDate, false); + addSinceLeakPeriodFilter(dbSession, project, builder); } } - if (project.getMainBranchProjectUuid() == null) { - builder.mainBranch(true); - } else { - builder.branchUuid(project.uuid()); - builder.mainBranch(false); - } + addMainBranchFilter(project, builder); } if (!wsRequest.getHotspotKeys().isEmpty()) { @@ -379,6 +374,15 @@ public class SearchAction implements HotspotsWsAction { wsRequest.getStatus().ifPresent(status -> builder.resolved(STATUS_REVIEWED.equals(status))); wsRequest.getResolution().ifPresent(resolution -> builder.resolutions(singleton(resolution))); + addSecurityStandardFilters(wsRequest, builder); + + IssueQuery query = builder.build(); + SearchOptions searchOptions = new SearchOptions() + .setPage(wsRequest.page, wsRequest.index); + return issueIndex.search(query, searchOptions); + } + + private static void addSecurityStandardFilters(WsRequest wsRequest, IssueQuery.Builder builder) { if (!wsRequest.getOwaspTop10().isEmpty()) { builder.owaspTop10(wsRequest.getOwaspTop10()); } @@ -388,18 +392,38 @@ public class SearchAction implements HotspotsWsAction { if (!wsRequest.getSonarsourceSecurity().isEmpty()) { builder.sonarsourceSecurity(wsRequest.getSonarsourceSecurity()); } - if (!wsRequest.getCwe().isEmpty()) { builder.cwe(wsRequest.getCwe()); } + } - IssueQuery query = builder.build(); - SearchOptions searchOptions = new SearchOptions() - .setPage(wsRequest.page, wsRequest.index); - return issueIndex.search(query, searchOptions); + private static void addMainBranchFilter(@NotNull ComponentDto project, IssueQuery.Builder builder) { + if (project.getMainBranchProjectUuid() == null) { + builder.mainBranch(true); + } else { + builder.branchUuid(project.uuid()); + builder.mainBranch(false); + } + } + + private void addSinceLeakPeriodFilter(DbSession dbSession, @NotNull ComponentDto project, IssueQuery.Builder builder) { + Optional<SnapshotDto> snapshot = dbClient.snapshotDao().selectLastAnalysisByComponentUuid(dbSession, project.uuid()); + + boolean isLastAnalysisUsingReferenceBranch = snapshot.map(SnapshotDto::getPeriodMode) + .orElse("").equals(REFERENCE_BRANCH.name()); + + if (isLastAnalysisUsingReferenceBranch) { + builder.newCodeOnReference(true); + } else { + var sinceDate = snapshot + .map(s -> longToDate(s.getPeriodDate())) + .orElseGet(() -> new Date(system2.now())); + + builder.createdAfter(sinceDate, false); + } } - private void addCreatedAfterByProjects(IssueQuery.Builder builder, DbSession dbSession, ComponentDto application) { + private void addSinceLeakPeriodFilterByProjects(IssueQuery.Builder builder, DbSession dbSession, ComponentDto application) { Set<String> projectUuids; if (application.getMainBranchProjectUuid() == null) { projectUuids = dbClient.applicationProjectsDao().selectProjects(dbSession, application.uuid()).stream() @@ -412,10 +436,23 @@ public class SearchAction implements HotspotsWsAction { } long now = system2.now(); - Map<String, IssueQuery.PeriodStart> leakByProjects = dbClient.snapshotDao().selectLastAnalysesByRootComponentUuids(dbSession, projectUuids).stream() - .collect(uniqueIndex(SnapshotDto::getComponentUuid, s -> new IssueQuery.PeriodStart(longToDate(s.getPeriodDate() == null ? now : s.getPeriodDate()), false))); + + List<SnapshotDto> snapshots = dbClient.snapshotDao().selectLastAnalysesByRootComponentUuids(dbSession, projectUuids); + + Set<String> newCodeReferenceByProjects = snapshots + .stream() + .filter(s -> !isNullOrEmpty(s.getPeriodMode()) && s.getPeriodMode().equals(REFERENCE_BRANCH.name())) + .map(SnapshotDto::getComponentUuid) + .collect(toSet()); + + Map<String, IssueQuery.PeriodStart> leakByProjects = snapshots + .stream() + .filter(s -> isNullOrEmpty(s.getPeriodMode()) || !s.getPeriodMode().equals(REFERENCE_BRANCH.name())) + .collect(uniqueIndex(SnapshotDto::getComponentUuid, s -> + new IssueQuery.PeriodStart(longToDate(s.getPeriodDate() == null ? now : s.getPeriodDate()), false))); builder.createdAfterByProjectUuids(leakByProjects); + builder.newCodeOnReferenceByProjectUuids(newCodeReferenceByProjects); } private void loadComponents(DbSession dbSession, SearchResponseData searchResponseData) { diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/hotspot/ws/SearchActionTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/hotspot/ws/SearchActionTest.java index bd676ba254c..d926f9b2fc5 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/hotspot/ws/SearchActionTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/hotspot/ws/SearchActionTest.java @@ -99,7 +99,9 @@ import static org.sonar.api.web.UserRole.USER; import static org.sonar.core.util.stream.MoreCollectors.uniqueIndex; import static org.sonar.db.component.ComponentTesting.newDirectory; import static org.sonar.db.component.ComponentTesting.newFileDto; +import static org.sonar.db.issue.IssueTesting.newCodeReferenceIssue; import static org.sonar.db.issue.IssueTesting.newIssue; +import static org.sonar.db.newcodeperiod.NewCodePeriodType.REFERENCE_BRANCH; @RunWith(DataProviderRunner.class) public class SearchActionTest { @@ -1485,6 +1487,47 @@ public class SearchActionTest { } @Test + public void returns_hotspots_on_the_leak_period_when_sinceLeakPeriod_is_true_and_branch_uses_reference_branch() { + ComponentDto project = dbTester.components().insertPublicProject(); + userSessionRule.registerComponents(project); + indexPermissions(); + ComponentDto file = dbTester.components().insertComponent(newFileDto(project)); + dbTester.components().insertSnapshot(project, t -> t.setPeriodMode(REFERENCE_BRANCH.name()).setPeriodParam("master")); + RuleDefinitionDto rule = newRule(SECURITY_HOTSPOT); + List<IssueDto> hotspotsInLeakPeriod = IntStream.range(0, 1 + RANDOM.nextInt(20)) + .mapToObj(i -> dbTester.issues().insertHotspot(rule, project, file, t -> t.setLine(i))) + .collect(toList()); + + hotspotsInLeakPeriod.stream().forEach(i -> dbTester.issues().insertNewCodeReferenceIssue(newCodeReferenceIssue(i))); + + List<IssueDto> hotspotsNotInLeakPeriod = IntStream.range(0, 1 + RANDOM.nextInt(20)) + .mapToObj(i -> dbTester.issues().insertHotspot(rule, project, file, t -> t.setLine(i))) + .collect(toList()); + indexIssues(); + + SearchWsResponse responseAll = newRequest(project) + .executeProtobuf(SearchWsResponse.class); + assertThat(responseAll.getHotspotsList()) + .extracting(SearchWsResponse.Hotspot::getKey) + .containsExactlyInAnyOrder(Stream.of( + hotspotsInLeakPeriod.stream(), + hotspotsNotInLeakPeriod.stream()) + .flatMap(t -> t) + .map(IssueDto::getKey) + .toArray(String[]::new)); + + SearchWsResponse responseOnLeak = newRequest(project, + t -> t.setParam("sinceLeakPeriod", "true")) + .executeProtobuf(SearchWsResponse.class); + assertThat(responseOnLeak.getHotspotsList()) + .extracting(SearchWsResponse.Hotspot::getKey) + .containsExactlyInAnyOrder(hotspotsInLeakPeriod + .stream() + .map(IssueDto::getKey) + .toArray(String[]::new)); + } + + @Test public void returns_nothing_when_sinceLeakPeriod_is_true_and_no_period_exists() { long referenceDate = 800_996_999_332L; |