From 564d295d94911eeea1e5ef192bd675c3ff1c406e Mon Sep 17 00:00:00 2001 From: Benjamin Campomenosi Date: Mon, 16 Oct 2023 09:28:51 +0200 Subject: [PATCH] SONAR-20664 migrate ExportIssueStep to MyBatis. --- .../projectexport/issue/ExportIssuesStep.java | 179 ++++++------------ .../sonar/db/project/ProjectExportMapper.java | 3 + .../sonar/db/project/ProjectExportMapper.xml | 59 +++++- 3 files changed, 120 insertions(+), 121 deletions(-) diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/issue/ExportIssuesStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/issue/ExportIssuesStep.java index e49c67498bb..d0b92faf001 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/issue/ExportIssuesStep.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectexport/issue/ExportIssuesStep.java @@ -22,14 +22,12 @@ package org.sonar.ce.task.projectexport.issue; import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; +import com.hazelcast.internal.util.MutableLong; import com.sonarsource.governance.projectdump.protobuf.ProjectDump; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; import java.util.List; import java.util.Objects; import java.util.Optional; -import javax.annotation.Nullable; +import org.apache.ibatis.cursor.Cursor; import org.slf4j.LoggerFactory; import org.sonar.api.rule.RuleKey; import org.sonar.ce.task.projectexport.component.ComponentRepository; @@ -41,37 +39,15 @@ import org.sonar.ce.task.projectexport.steps.DumpWriter; import org.sonar.ce.task.projectexport.steps.ProjectHolder; import org.sonar.ce.task.projectexport.steps.StreamWriter; import org.sonar.ce.task.step.ComputationStep; -import org.sonar.db.DatabaseUtils; import org.sonar.db.DbClient; import org.sonar.db.DbSession; +import org.sonar.db.issue.IssueDto; +import org.sonar.db.project.ProjectExportMapper; import org.sonar.db.protobuf.DbIssues; import static java.lang.String.format; -import static org.sonar.ce.task.projectexport.util.ResultSetUtils.defaultIfNull; -import static org.sonar.ce.task.projectexport.util.ResultSetUtils.emptyIfNull; public class ExportIssuesStep implements ComputationStep { - private static final String RULE_STATUS_REMOVED = "REMOVED"; - private static final String ISSUE_STATUS_CLOSED = "CLOSED"; - - // ordered by rule_id to reduce calls to RuleRepository - private static final String QUERY = "select" + - " i.kee, r.uuid, r.plugin_rule_key, r.plugin_name, i.issue_type," + - " i.component_uuid, i.message, i.line, i.checksum, i.status," + - " i.resolution, i.severity, i.manual_severity, i.gap, effort," + - " i.assignee, i.author_login, i.tags, i.issue_creation_date," + - " i.issue_update_date, i.issue_close_date, i.locations, i.project_uuid," + - " i.rule_description_context_key, i.message_formattings, i.code_variants, " + - " ii.software_quality, ii.severity" + - " from issues i" + - " join rules r on r.uuid = i.rule_uuid and r.status <> ?" + - " join components p on p.uuid = i.project_uuid" + - " join project_branches pb on pb.uuid = p.uuid" + - " left outer join issues_impacts ii on i.kee = ii.issue_key" + - " where pb.project_uuid = ? and pb.branch_type = 'BRANCH' and pb.exclude_from_purge=?" + - " and i.status <> ?" + - " order by" + - " i.rule_uuid asc, i.kee, i.created_at asc"; private final DbClient dbClient; private final ProjectHolder projectHolder; @@ -95,122 +71,89 @@ public class ExportIssuesStep implements ComputationStep { @Override public void execute(Context context) { - long count = 0; + MutableLong count = MutableLong.valueOf(0); try ( StreamWriter output = dumpWriter.newStreamWriter(DumpElement.ISSUES); DbSession dbSession = dbClient.openSession(false); - PreparedStatement stmt = createStatement(dbSession); - ResultSet rs = stmt.executeQuery()) { - ProjectDump.Issue.Builder builder = ProjectDump.Issue.newBuilder(); - ProjectDump.Issue previousIssue = null; - while (rs.next()) { - String issueUuid = rs.getString(1); - if ((!rs.isFirst() && !previousIssue.getUuid().equals(issueUuid))) { - output.write(previousIssue); - count++; - } - ProjectDump.Issue issue = mergeIssue(previousIssue, builder, rs); - if (rs.isLast()) { + Cursor issueDtoCursor = dbSession.getMapper(ProjectExportMapper.class).scrollIssueForExport(projectHolder.projectDto().getUuid())) { + ProjectDump.Issue.Builder issueBuilder = ProjectDump.Issue.newBuilder(); + issueDtoCursor + .forEach(issueDto -> { + ProjectDump.Issue issue = convertToIssue(issueDto, issueBuilder); output.write(issue); - count++; - } - previousIssue = issue; - } - - LoggerFactory.getLogger(getClass()).debug("{} issues exported", count); + count.getAndInc(); + }); + LoggerFactory.getLogger(getClass()).debug("{} issues exported", count.value); } catch (Exception e) { - throw new IllegalStateException(format("Issue export failed after processing %d issues successfully", count), e); + throw new IllegalStateException(format("Issue export failed after processing %d issues successfully", count.value), e); } } - private PreparedStatement createStatement(DbSession dbSession) throws SQLException { - PreparedStatement stmt = dbClient.getMyBatis().newScrollingSelectStatement(dbSession, QUERY); - try { - stmt.setString(1, RULE_STATUS_REMOVED); - stmt.setString(2, projectHolder.projectDto().getUuid()); - stmt.setBoolean(3, true); - stmt.setString(4, ISSUE_STATUS_CLOSED); - return stmt; - } catch (Exception t) { - DatabaseUtils.closeQuietly(stmt); - throw t; - } - } + private ProjectDump.Issue convertToIssue(IssueDto issueDto, ProjectDump.Issue.Builder builder) { - private ProjectDump.Issue mergeIssue(@Nullable ProjectDump.Issue previousIssue, ProjectDump.Issue.Builder builder, ResultSet rs) throws SQLException { - builder.clear(); - String issueUuid = rs.getString(1); - setRule(builder, rs); + String ruleRef = registerRule(issueDto); builder - .setUuid(issueUuid) - .setType(rs.getInt(5)) - .setComponentRef(componentRepository.getRef(rs.getString(6))) - .setMessage(emptyIfNull(rs, 7)) - .setLine(rs.getInt(8)) - .setChecksum(emptyIfNull(rs, 9)) - .setStatus(emptyIfNull(rs, 10)) - .setResolution(emptyIfNull(rs, 11)) - .setSeverity(emptyIfNull(rs, 12)) - .setManualSeverity(rs.getBoolean(13)) - .setGap(defaultIfNull(rs, 14, IssueDumpElement.NO_GAP)) - .setEffort(defaultIfNull(rs, 15, IssueDumpElement.NO_EFFORT)) - .setAssignee(emptyIfNull(rs, 16)) - .setAuthor(emptyIfNull(rs, 17)) - .setTags(emptyIfNull(rs, 18)) - .setIssueCreatedAt(rs.getLong(19)) - .setIssueUpdatedAt(rs.getLong(20)) - .setIssueClosedAt(rs.getLong(21)) - .setProjectUuid(rs.getString(23)) - .setCodeVariants(emptyIfNull(rs, 26)); - Optional.ofNullable(rs.getString(24)).ifPresent(builder::setRuleDescriptionContextKey); - setLocations(builder, rs, issueUuid); - setMessageFormattings(builder, rs, issueUuid); - - mergeImpacts(builder, rs, previousIssue, issueUuid); + .clear() + .setRuleRef(ruleRef) + .setUuid(issueDto.getKee()) + .setComponentRef(componentRepository.getRef(issueDto.getComponentUuid())) + .setType(issueDto.getType()) + .setMessage(Optional.of(issueDto).map(IssueDto::getMessage).orElse("")) + .setLine(Optional.of(issueDto).map(IssueDto::getLine).orElse(0)) + .setChecksum(Optional.of(issueDto).map(IssueDto::getChecksum).orElse("")) + .setStatus(Optional.of(issueDto).map(IssueDto::getStatus).orElse("")) + .setResolution(Optional.of(issueDto).map(IssueDto::getResolution).orElse("")) + .setSeverity(Optional.of(issueDto).map(IssueDto::getSeverity).orElse("")) + .setManualSeverity(issueDto.isManualSeverity()) + .setGap(Optional.of(issueDto).map(IssueDto::getGap).orElse(IssueDumpElement.NO_GAP)) + .setEffort(Optional.of(issueDto).map(IssueDto::getEffort).orElse(IssueDumpElement.NO_EFFORT)) + .setAssignee(Optional.of(issueDto).map(IssueDto::getAssigneeUuid).orElse("")) + .setAuthor(Optional.of(issueDto).map(IssueDto::getAuthorLogin).orElse("")) + .setTags(Optional.of(issueDto).map(IssueDto::getTagsString).orElse("")) + .setIssueCreatedAt(Optional.of(issueDto).map(IssueDto::getIssueCreationTime).orElse(0L)) + .setIssueUpdatedAt(Optional.of(issueDto).map(IssueDto::getIssueUpdateTime).orElse(0L)) + .setIssueClosedAt(Optional.of(issueDto).map(IssueDto::getIssueCloseTime).orElse(0L)) + .setProjectUuid(issueDto.getProjectUuid()) + .setCodeVariants(Optional.of(issueDto).map(IssueDto::getCodeVariantsString).orElse("")); + setLocations(builder, issueDto); + setMessageFormattings(builder, issueDto); + mergeImpacts(builder, issueDto); return builder.build(); } - private static void mergeImpacts(ProjectDump.Issue.Builder builder, ResultSet rs, @Nullable ProjectDump.Issue previousIssue, String issueUuid) - throws SQLException { - String softwareQualityFromDatabase = rs.getString(27); - if (softwareQualityFromDatabase == null) { - return; - } - ProjectDump.Impact impact = ProjectDump.Impact.newBuilder() - .setSoftwareQuality(ProjectDump.SoftwareQuality.valueOf(softwareQualityFromDatabase)) - .setSeverity(ProjectDump.Severity.valueOf(rs.getString(28))) - .build(); - - builder.addImpacts(impact); - if (previousIssue != null && previousIssue.getUuid().equals(issueUuid)) { - builder.addAllImpacts(previousIssue.getImpactsList()); - } + private static void mergeImpacts(ProjectDump.Issue.Builder builder, IssueDto issueDto) { + issueDto.getImpacts() + .stream() + .map(impactDto -> ProjectDump.Impact.newBuilder() + .setSoftwareQuality(ProjectDump.SoftwareQuality.valueOf(impactDto.getSoftwareQuality().name())) + .setSeverity(ProjectDump.Severity.valueOf(impactDto.getSeverity().name())) + .build()) + .forEach(builder::addImpacts); } - private void setRule(ProjectDump.Issue.Builder builder, ResultSet rs) throws SQLException { - String ruleUuid = rs.getString(2); - String ruleKey = rs.getString(3); - String repositoryKey = rs.getString(4); - builder.setRuleRef(ruleRegistrar.register(ruleUuid, repositoryKey, ruleKey).ref()); + private String registerRule(IssueDto issueDto) { + String ruleUuid = issueDto.getRuleUuid(); + RuleKey ruleKey = issueDto.getRuleKey(); + return ruleRegistrar.register(ruleUuid, ruleKey).ref(); } - private static void setLocations(ProjectDump.Issue.Builder builder, ResultSet rs, String issueUuid) throws SQLException { + private static void setLocations(ProjectDump.Issue.Builder builder, IssueDto issueDto) { try { - byte[] bytes = rs.getBytes(22); + byte[] bytes = issueDto.getLocations(); if (bytes != null) { // fail fast, ensure we can read data from DB DbIssues.Locations.parseFrom(bytes); builder.setLocations(ByteString.copyFrom(bytes)); } } catch (InvalidProtocolBufferException e) { - throw new IllegalStateException(format("Fail to read locations from DB for issue %s", issueUuid), e); + throw new IllegalStateException(format("Fail to read locations from DB for issue %s", issueDto.getKee()), e); } } - private static void setMessageFormattings(ProjectDump.Issue.Builder builder, ResultSet rs, String issueUuid) throws SQLException { + private static void setMessageFormattings(ProjectDump.Issue.Builder builder, IssueDto issueDto) { try { - byte[] bytes = rs.getBytes(25); + byte[] bytes = issueDto.getMessageFormattings(); if (bytes != null) { // fail fast, ensure we can read data from DB DbIssues.MessageFormattings messageFormattings = DbIssues.MessageFormattings.parseFrom(bytes); @@ -219,7 +162,7 @@ public class ExportIssuesStep implements ComputationStep { } } } catch (InvalidProtocolBufferException e) { - throw new IllegalStateException(format("Fail to read message formattings from DB for issue %s", issueUuid), e); + throw new IllegalStateException(format("Fail to read message formattings from DB for issue %s", issueDto.getKee()), e); } } @@ -242,11 +185,11 @@ public class ExportIssuesStep implements ComputationStep { this.ruleRepository = ruleRepository; } - public Rule register(String ruleUuid, String repositoryKey, String ruleKey) { + public Rule register(String ruleUuid, RuleKey ruleKey) { if (Objects.equals(previousRuleUuid, ruleUuid)) { return previousRule; } - return lookup(ruleUuid, RuleKey.of(repositoryKey, ruleKey)); + return lookup(ruleUuid, ruleKey); } private Rule lookup(String ruleUuid, RuleKey ruleKey) { diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectExportMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectExportMapper.java index 2641c838ff1..95b50177829 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectExportMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectExportMapper.java @@ -24,6 +24,7 @@ import org.apache.ibatis.annotations.Param; import org.apache.ibatis.cursor.Cursor; import org.sonar.db.component.BranchDto; import org.sonar.db.component.ProjectLinkDto; +import org.sonar.db.issue.IssueDto; import org.sonar.db.newcodeperiod.NewCodePeriodDto; import org.sonar.db.property.PropertyDto; import org.sonar.db.rule.RuleDto; @@ -40,4 +41,6 @@ public interface ProjectExportMapper { Cursor scrollAdhocRulesForExport(@Param("projectUuid") String projectUuid); + Cursor scrollIssueForExport(@Param("projectUuid") String projectUuid); + } diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/project/ProjectExportMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/project/ProjectExportMapper.xml index 9e4b0eb114a..cc31ef0639c 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/project/ProjectExportMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/project/ProjectExportMapper.xml @@ -55,7 +55,8 @@ (ncp.branch_uuid is null OR (pb.branch_type='BRANCH' AND pb.exclude_from_purge=${_true})) - + + + + -- 2.39.5