From c5f6c08551a945e6cc7c2450b0fb7ae4e795ba90 Mon Sep 17 00:00:00 2001 From: Jacek Poreda Date: Mon, 7 Aug 2023 14:53:59 +0200 Subject: [PATCH] SONAR-20021 Add filter and facets for Software Quality, Severity and Clean Code Attribute Category - Refactor issue indexer to use MyBatis mapping with cursor --- .../java/org/sonar/db/issue/IssueDaoIT.java | 36 ++ .../org/sonar/db/issue/IndexedIssueDto.java | 316 ++++++++++++++++++ .../java/org/sonar/db/issue/IssueDao.java | 8 + .../java/org/sonar/db/issue/IssueMapper.java | 3 + .../org/sonar/db/issue/IssueMapper.xml | 69 ++++ .../sonar/db/issue/IndexedIssueDtoTest.java | 89 +++++ .../server/issue/index/IssueIndexerIT.java | 9 + .../issue/index/IssueIteratorFactoryIT.java | 1 + .../sonar/server/issue/index/IssueDoc.java | 32 ++ .../issue/index/IssueIndexDefinition.java | 11 + .../index/IssueIteratorForSingleChunk.java | 282 ++++++---------- .../measure/index/ProjectMeasuresDoc.java | 1 - .../sonar/server/issue/index/IssueIndex.java | 131 +++++++- .../sonar/server/issue/index/IssueQuery.java | 36 ++ .../issue/index/IssueIndexFacetsTest.java | 178 +++++++++- .../issue/index/IssueIndexFiltersTest.java | 113 +++++-- .../ws/client/issue/IssuesWsParameters.java | 5 + 17 files changed, 1083 insertions(+), 237 deletions(-) create mode 100644 server/sonar-db-dao/src/main/java/org/sonar/db/issue/IndexedIssueDto.java create mode 100644 server/sonar-db-dao/src/test/java/org/sonar/db/issue/IndexedIssueDtoTest.java diff --git a/server/sonar-db-dao/src/it/java/org/sonar/db/issue/IssueDaoIT.java b/server/sonar-db-dao/src/it/java/org/sonar/db/issue/IssueDaoIT.java index d9f8ac85ee9..0a4a79a5e86 100644 --- a/server/sonar-db-dao/src/it/java/org/sonar/db/issue/IssueDaoIT.java +++ b/server/sonar-db-dao/src/it/java/org/sonar/db/issue/IssueDaoIT.java @@ -22,12 +22,14 @@ package org.sonar.db.issue; import java.util.Collection; import java.util.Collections; import java.util.Date; +import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.IntStream; import java.util.stream.Stream; import javax.annotation.Nullable; +import org.apache.ibatis.cursor.Cursor; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -198,6 +200,40 @@ public class IssueDaoIT { tuple(Severity.LOW, SoftwareQuality.SECURITY)); } + @Test + public void scrollIndexationIssues_shouldReturnDto() { + ComponentDto project = db.components().insertPrivateProject().getMainBranchComponent(); + RuleDto rule = db.rules().insert(r -> r.setRepositoryKey("java").setLanguage("java") + .addDefaultImpact(new ImpactDto() + .setUuid(UuidFactoryFast.getInstance().create()) + .setSoftwareQuality(SoftwareQuality.RELIABILITY) + .setSeverity(Severity.MEDIUM))); + + ComponentDto branchA = db.components().insertProjectBranch(project, b -> b.setKey("branchA")); + ComponentDto fileA = db.components().insertComponent(newFileDto(branchA)); + + IntStream.range(0, 100).forEach(i -> insertBranchIssue(branchA, fileA, rule, "A" + i, STATUS_OPEN, 1_340_000_000_000L)); + + Cursor issues = underTest.scrollIssuesForIndexation(db.getSession(), null, null); + + Iterator iterator = issues.iterator(); + int issueCount = 0; + while (iterator.hasNext()) { + IndexedIssueDto next = iterator.next(); + assertThat(next.getRuleDefaultImpacts()).hasSize(2) + .extracting(ImpactDto::getSoftwareQuality, ImpactDto::getSeverity) + .containsExactlyInAnyOrder( + tuple(SoftwareQuality.RELIABILITY, Severity.MEDIUM), + tuple(SoftwareQuality.MAINTAINABILITY, Severity.HIGH)); + assertThat(next.getImpacts()) + .extracting(ImpactDto::getSoftwareQuality, ImpactDto::getSeverity) + .containsExactlyInAnyOrder( + tuple(SoftwareQuality.MAINTAINABILITY, Severity.HIGH)); + issueCount++; + } + assertThat(issueCount).isEqualTo(100); + } + @Test public void selectIssueKeysByComponentUuid() { // contains I1 and I2 diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/issue/IndexedIssueDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/issue/IndexedIssueDto.java new file mode 100644 index 00000000000..dc502006d56 --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/issue/IndexedIssueDto.java @@ -0,0 +1,316 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.issue; + +import java.util.Collections; +import java.util.EnumMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import org.sonar.api.issue.impact.Severity; +import org.sonar.api.issue.impact.SoftwareQuality; + +public final class IndexedIssueDto { + private String issueKey = null; + private String assignee = null; + private Integer line = null; + private String resolution = null; + private String cleanCodeAttribute = null; + private String severity = null; + private String status = null; + private Long effort = null; + private String authorLogin = null; + private Long issueCloseDate = null; + private Long issueCreationDate = null; + private Long issueUpdateDate = null; + private String ruleUuid = null; + private String language = null; + private String componentUuid = null; + private String path = null; + private String scope = null; + private String branchUuid = null; + private boolean isMain = false; + private String projectUuid = null; + private String tags = null; + private Integer issueType = null; + private String securityStandards = null; + private String qualifier = null; + private boolean isNewCodeReferenceIssue = false; + private String codeVariants = null; + + private Set impacts = new HashSet<>(); + private Set ruleDefaultImpacts = new HashSet<>(); + + public IndexedIssueDto() { + // empty constructor + } + + public String getIssueKey() { + return issueKey; + } + + public IndexedIssueDto setIssueKey(String issueKey) { + this.issueKey = issueKey; + return this; + } + + public String getAssignee() { + return assignee; + } + + public IndexedIssueDto setAssignee(String assignee) { + this.assignee = assignee; + return this; + } + + public Integer getLine() { + return line; + } + + public IndexedIssueDto setLine(Integer line) { + this.line = line; + return this; + } + + public String getResolution() { + return resolution; + } + + public IndexedIssueDto setResolution(String resolution) { + this.resolution = resolution; + return this; + } + + public String getSeverity() { + return severity; + } + + public IndexedIssueDto setSeverity(String severity) { + this.severity = severity; + return this; + } + + public String getStatus() { + return status; + } + + public IndexedIssueDto setStatus(String status) { + this.status = status; + return this; + } + + public Long getEffort() { + return effort; + } + + public IndexedIssueDto setEffort(Long effort) { + this.effort = effort; + return this; + } + + public String getAuthorLogin() { + return authorLogin; + } + + public IndexedIssueDto setAuthorLogin(String authorLogin) { + this.authorLogin = authorLogin; + return this; + } + + public Long getIssueCloseDate() { + return issueCloseDate; + } + + public IndexedIssueDto setIssueCloseDate(Long issueCloseDate) { + this.issueCloseDate = issueCloseDate; + return this; + } + + public Long getIssueCreationDate() { + return issueCreationDate; + } + + public IndexedIssueDto setIssueCreationDate(Long issueCreationDate) { + this.issueCreationDate = issueCreationDate; + return this; + } + + public Long getIssueUpdateDate() { + return issueUpdateDate; + } + + public IndexedIssueDto setIssueUpdateDate(Long issueUpdateDate) { + this.issueUpdateDate = issueUpdateDate; + return this; + } + + public String getRuleUuid() { + return ruleUuid; + } + + public IndexedIssueDto setRuleUuid(String ruleUuid) { + this.ruleUuid = ruleUuid; + return this; + } + + public String getLanguage() { + return language; + } + + public IndexedIssueDto setLanguage(String language) { + this.language = language; + return this; + } + + public String getComponentUuid() { + return componentUuid; + } + + public IndexedIssueDto setComponentUuid(String componentUuid) { + this.componentUuid = componentUuid; + return this; + } + + public String getPath() { + return path; + } + + public IndexedIssueDto setPath(String path) { + this.path = path; + return this; + } + + public String getScope() { + return scope; + } + + public IndexedIssueDto setScope(String scope) { + this.scope = scope; + return this; + } + + public String getBranchUuid() { + return branchUuid; + } + + public IndexedIssueDto setBranchUuid(String branchUuid) { + this.branchUuid = branchUuid; + return this; + } + + public boolean isMain() { + return isMain; + } + + public IndexedIssueDto setIsMain(boolean isMain) { + this.isMain = isMain; + return this; + } + + public String getProjectUuid() { + return projectUuid; + } + + public IndexedIssueDto setProjectUuid(String projectUuid) { + this.projectUuid = projectUuid; + return this; + } + + public String getTags() { + return tags; + } + + public IndexedIssueDto setTags(String tags) { + this.tags = tags; + return this; + } + + @Deprecated + public Integer getIssueType() { + return issueType; + } + + @Deprecated + public IndexedIssueDto setIssueType(Integer issueType) { + this.issueType = issueType; + return this; + } + + public String getSecurityStandards() { + return securityStandards; + } + + public IndexedIssueDto setSecurityStandards(String securityStandards) { + this.securityStandards = securityStandards; + return this; + } + + public String getQualifier() { + return qualifier; + } + + public IndexedIssueDto setQualifier(String qualifier) { + this.qualifier = qualifier; + return this; + } + + public boolean isNewCodeReferenceIssue() { + return isNewCodeReferenceIssue; + } + + public IndexedIssueDto setNewCodeReferenceIssue(boolean newCodeReferenceIssue) { + isNewCodeReferenceIssue = newCodeReferenceIssue; + return this; + } + + public String getCodeVariants() { + return codeVariants; + } + + public IndexedIssueDto setCodeVariants(String codeVariants) { + this.codeVariants = codeVariants; + return this; + } + + public Set getImpacts() { + return impacts; + } + + public Set getRuleDefaultImpacts() { + return ruleDefaultImpacts; + } + + + public Map getEffectiveImpacts() { + EnumMap effectiveImpacts = new EnumMap<>(SoftwareQuality.class); + ruleDefaultImpacts.forEach(impact -> effectiveImpacts.put(impact.getSoftwareQuality(), impact.getSeverity())); + impacts.forEach(impact -> effectiveImpacts.put(impact.getSoftwareQuality(), impact.getSeverity())); + return Collections.unmodifiableMap(effectiveImpacts); + } + + public String getCleanCodeAttribute() { + return cleanCodeAttribute; + } + + public IndexedIssueDto setCleanCodeAttribute(String cleanCodeAttribute) { + this.cleanCodeAttribute = cleanCodeAttribute; + return this; + } +} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/issue/IssueDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/issue/IssueDao.java index 6efd896f392..af95135fb9f 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/issue/IssueDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/issue/IssueDao.java @@ -23,6 +23,9 @@ import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.Set; +import javax.annotation.Nullable; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.cursor.Cursor; import org.sonar.db.Dao; import org.sonar.db.DbSession; import org.sonar.db.Pagination; @@ -91,6 +94,11 @@ public class IssueDao implements Dao { return mapper(dbSession).selectIssueGroupsByComponent(component, leakPeriodBeginningDate); } + public Cursor scrollIssuesForIndexation(DbSession dbSession, @Nullable @Param("branchUuid") String branchUuid, + @Nullable @Param("issueKeys") Collection issueKeys) { + return mapper(dbSession).scrollIssuesForIndexation(branchUuid, issueKeys); + } + public void insert(DbSession session, IssueDto dto) { mapper(session).insert(dto); updateIssueImpacts(dto, mapper(session)); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/issue/IssueMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/issue/IssueMapper.java index 579c69c13f6..93951e438e6 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/issue/IssueMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/issue/IssueMapper.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Set; import javax.annotation.Nullable; import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.cursor.Cursor; import org.apache.ibatis.session.ResultHandler; import org.sonar.db.Pagination; import org.sonar.db.component.ComponentDto; @@ -72,6 +73,8 @@ public interface IssueMapper { void scrollClosedByComponentUuid(@Param("componentUuid") String componentUuid, @Param("closeDateAfter") long closeDateAfter, ResultHandler handler); + Cursor scrollIssuesForIndexation(@Nullable @Param("branchUuid") String branchUuid, @Nullable @Param("issueKeys") Collection issueKeys); + Collection selectIssueGroupsByComponent(@Param("component") ComponentDto component, @Param("leakPeriodBeginningDate") long leakPeriodBeginningDate); List selectByBranch(@Param("keys") Set keys, @Nullable @Param("changedSince") Long changedSince); diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/issue/IssueMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/issue/IssueMapper.xml index e1966bf9aa6..17889d58ee3 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/issue/IssueMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/issue/IssueMapper.xml @@ -317,6 +317,75 @@ i.kee, ic.issue_change_creation_date desc + + + + + + + + + + + + + + + + +