From 9806fd384d27548e65dad887fa1bf2a3d0a5320b Mon Sep 17 00:00:00 2001 From: "antoine.vinot" Date: Fri, 6 Sep 2024 15:12:28 +0200 Subject: [PATCH] SONAR-22914 Compute CVEs received from scanner report --- .../step/PersistCveStepIT.java | 156 ++++++++++++++++++ .../step/PersistIssuesStepDependencyIT.java | 126 ++++++++++++++ .../batch/BatchReportReader.java | 2 + .../batch/BatchReportReaderImpl.java | 6 + .../issue/TrackerRawInputFactory.java | 1 + .../projectanalysis/step/PersistCveStep.java | 139 ++++++++++++++++ .../step/PersistIssuesStep.java | 16 +- .../step/ReportComputationSteps.java | 1 + .../util/cache/ProtobufIssueDiskCache.java | 3 +- .../src/main/protobuf/issue_cache.proto | 95 +++++------ .../src/main/protobuf/project_dump.proto | 14 +- .../batch/BatchReportReaderImplTest.java | 25 +++ .../batch/BatchReportReaderRule.java | 11 ++ .../org/sonar/db/dependency/CveCweDaoIT.java | 19 +++ .../org/sonar/db/dependency/CveDaoIT.java | 13 ++ .../org/sonar/db/dependency/CveCweDao.java | 5 + .../org/sonar/db/dependency/CveCweMapper.java | 2 + .../java/org/sonar/db/dependency/CveDao.java | 4 + .../org/sonar/db/dependency/CveMapper.java | 2 + .../java/org/sonar/db/issue/IssueDto.java | 6 +- .../org/sonar/db/dependency/CveCweMapper.xml | 5 + .../org/sonar/db/dependency/CveMapper.xml | 19 ++- .../org/sonar/core/issue/DefaultIssue.java | 11 ++ 23 files changed, 620 insertions(+), 61 deletions(-) create mode 100644 server/sonar-ce-task-projectanalysis/src/it/java/org/sonar/ce/task/projectanalysis/step/PersistCveStepIT.java create mode 100644 server/sonar-ce-task-projectanalysis/src/it/java/org/sonar/ce/task/projectanalysis/step/PersistIssuesStepDependencyIT.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistCveStep.java diff --git a/server/sonar-ce-task-projectanalysis/src/it/java/org/sonar/ce/task/projectanalysis/step/PersistCveStepIT.java b/server/sonar-ce-task-projectanalysis/src/it/java/org/sonar/ce/task/projectanalysis/step/PersistCveStepIT.java new file mode 100644 index 00000000000..ad030ca4d77 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/it/java/org/sonar/ce/task/projectanalysis/step/PersistCveStepIT.java @@ -0,0 +1,156 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.step; + +import java.util.List; +import java.util.Set; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.sonar.api.utils.System2; +import org.sonar.ce.task.projectanalysis.batch.BatchReportReaderRule; +import org.sonar.ce.task.step.TestComputationStepContext; +import org.sonar.core.util.UuidFactory; +import org.sonar.core.util.UuidFactoryImpl; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.DbTester; +import org.sonar.db.dependency.CveCweDto; +import org.sonar.db.dependency.CveDto; +import org.sonar.scanner.protocol.output.ScannerReport.Cve; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +class PersistCveStepIT { + + @RegisterExtension + private final DbTester db = DbTester.create(System2.INSTANCE); + + @RegisterExtension + private final BatchReportReaderRule batchReportReader = new BatchReportReaderRule(); + + private final DbSession dbSession = db.getSession(); + private final DbClient dbClient = db.getDbClient(); + private final UuidFactory uuidFactory = UuidFactoryImpl.INSTANCE; + + private PersistCveStep persistCveStep; + + @BeforeEach + void setUp() { + persistCveStep = new PersistCveStep(batchReportReader, dbClient, uuidFactory, System2.INSTANCE); + } + + @Test + void getDescription_shouldReturnStepDescription() { + assertThat(persistCveStep.getDescription()).isEqualTo("Persist CVEs"); + } + + @Test + void execute_shouldInsertNewCVEs() { + Cve cve1 = buildCve("1").build(); + Cve cve2 = buildCve("2").build(); + batchReportReader.putCves(List.of(cve1, cve2)); + + persistCveStep.execute(new TestComputationStepContext()); + + assertCvePersistedInDatabase(cve1); + assertCvePersistedInDatabase(cve2); + } + + private void assertCvePersistedInDatabase(Cve cve) { + CveDto cveDto = dbClient.cveDao().selectById(dbSession, cve.getCveId()) + .orElseGet(() -> fail(String.format("CVE with id %s not found", cve.getCveId()))); + assertThat(cveDto.id()).isEqualTo(cve.getCveId()); + assertThat(cveDto.description()).isEqualTo(cve.getDescription()); + assertThat(cveDto.cvssScore()).isEqualTo(cve.getCvssScore()); + assertThat(cveDto.epssScore()).isEqualTo(cve.getEpssScore()); + assertThat(cveDto.epssPercentile()).isEqualTo(cve.getEpssPercentile()); + assertThat(cveDto.publishedAt()).isEqualTo(cve.getPublishedDate()); + assertThat(cveDto.lastModifiedAt()).isEqualTo(cve.getLastModifiedDate()); + assertThat(cveDto.uuid()).isNotBlank(); + assertThat(cveDto.createdAt()).isNotNull(); + assertThat(cveDto.updatedAt()).isNotNull(); + } + + @Test + void execute_shoudUpdateExistingCves() { + dbClient.cveDao().insert(dbSession, new CveDto("cve-uuid-1", "CVE-1", "Old description 1", 0.0F, 0.0F, 0.0F, 0L, 0L, 0L, 0L)); + dbClient.cveDao().insert(dbSession, new CveDto("cve-uuid-2", "CVE-2", "Old description 2", 0.0F, 0.0F, 0.0F, 0L, 0L, 0L, 0L)); + db.commit(); + Cve cve1 = buildCve("1").build(); + Cve cve2 = buildCve("2").build(); + batchReportReader.putCves(List.of(cve1, cve2)); + + persistCveStep.execute(new TestComputationStepContext()); + + assertThat(db.countRowsOfTable(dbSession, "cves")).isEqualTo(2); + assertCvePersistedInDatabase(cve1); + assertCvePersistedInDatabase(cve2); + } + + @Test + void execute_shouldInsertCwes_whenNewCVEs() { + Cve cve1 = buildCve("1").addCwe("CWE-11").addCwe("CWE-12").build(); + Cve cve2 = buildCve("2").addCwe("CWE-11").build(); + batchReportReader.putCves(List.of(cve1, cve2)); + + persistCveStep.execute(new TestComputationStepContext()); + + assertCveHasExactlyCwes(cve1, "CWE-11", "CWE-12"); + assertCveHasExactlyCwes(cve2, "CWE-11"); + } + + @Test + void execute_shouldUpdateExistingCwesAndInsertNewOnes_whenUpdatingCVEs() { + dbClient.cveDao().insert(dbSession, new CveDto("cve-uuid-1", "CVE-1", "Old description 1", 0.0F, 0.0F, 0.0F, 0L, 0L, 0L, 0L)); + dbClient.cveCweDao().insert(dbSession, new CveCweDto("cve-uuid-1", "CWE-1")); + dbClient.cveCweDao().insert(dbSession, new CveCweDto("cve-uuid-1", "CWE-2")); + db.commit(); + Cve cve = buildCve("1").addCwe("CWE-2").addCwe("CWE-3").build(); + batchReportReader.putCves(List.of(cve)); + + persistCveStep.execute(new TestComputationStepContext()); + + assertCveHasExactlyCwes(cve, "CWE-2", "CWE-3"); + } + + private void assertCveHasExactlyCwes(Cve cve, String... cwes) { + Set cweInDb = dbClient.cveCweDao().selectByCveUuid(dbSession, getCveUuid(cve.getCveId())); + assertThat(cweInDb).containsExactlyInAnyOrder(cwes); + } + + private String getCveUuid(String cveId) { + return dbClient.cveDao().selectById(dbSession, cveId) + .map(CveDto::uuid) + .orElseGet(() -> fail("CVE not found")); + } + + private static Cve.Builder buildCve(String suffix) { + return Cve.newBuilder() + .setCveId("CVE-"+suffix) + .setCvssScore(7.5F) + .setEpssScore(0.1F) + .setEpssPercentile(0.4F) + .setDescription("Some CVE description "+suffix) + .setLastModifiedDate(5L) + .setPublishedDate(4L); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/it/java/org/sonar/ce/task/projectanalysis/step/PersistIssuesStepDependencyIT.java b/server/sonar-ce-task-projectanalysis/src/it/java/org/sonar/ce/task/projectanalysis/step/PersistIssuesStepDependencyIT.java new file mode 100644 index 00000000000..084a8763ae7 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/it/java/org/sonar/ce/task/projectanalysis/step/PersistIssuesStepDependencyIT.java @@ -0,0 +1,126 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.step; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Date; +import java.util.List; +import java.util.Map; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.io.TempDir; +import org.sonar.api.rule.RuleKey; +import org.sonar.api.utils.System2; +import org.sonar.ce.task.projectanalysis.batch.BatchReportReaderRule; +import org.sonar.ce.task.projectanalysis.issue.AdHocRuleCreator; +import org.sonar.ce.task.projectanalysis.issue.ProtoIssueCache; +import org.sonar.ce.task.projectanalysis.issue.RuleRepositoryImpl; +import org.sonar.ce.task.projectanalysis.issue.UpdateConflictResolver; +import org.sonar.ce.task.projectanalysis.period.Period; +import org.sonar.ce.task.projectanalysis.period.PeriodHolderRule; +import org.sonar.ce.task.step.ComputationStep.Context; +import org.sonar.ce.task.step.TestComputationStepContext; +import org.sonar.core.issue.DefaultIssue; +import org.sonar.core.util.UuidFactoryImpl; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.DbTester; +import org.sonar.db.component.ComponentDto; +import org.sonar.db.dependency.CveDto; +import org.sonar.db.newcodeperiod.NewCodePeriodType; +import org.sonar.db.rule.RuleDto; +import org.sonar.db.rule.RuleTesting; +import org.sonar.scanner.protocol.output.ScannerReport; +import org.sonar.server.issue.IssueStorage; + +import static java.nio.file.Files.createTempFile; +import static org.mockito.Mockito.mock; +import static org.sonar.api.issue.IssueStatus.OPEN; +import static org.sonar.api.rules.RuleType.VULNERABILITY; + +class PersistIssuesStepDependencyIT { + + @RegisterExtension + private final DbTester db = DbTester.create(); + @RegisterExtension + private final BatchReportReaderRule reportReader = new BatchReportReaderRule(); + @RegisterExtension + private final PeriodHolderRule periodHolder = new PeriodHolderRule(); + @TempDir + private Path tempDir; + + private final DbSession dbSession = db.getSession(); + private final DbClient dbClient = db.getDbClient(); + + private ProtoIssueCache protoIssueCache; + + private PersistIssuesStep persistIssuesStep; + + private ComponentDto project; + RuleDto scaRule; + private CveDto cve; + + @BeforeEach + public void setUp() throws IOException { + periodHolder.setPeriod(new Period(NewCodePeriodType.NUMBER_OF_DAYS.name(), "10", 1000L)); + protoIssueCache = new ProtoIssueCache(createTempFile(tempDir, null, null).toFile(), System2.INSTANCE); + reportReader.setMetadata(ScannerReport.Metadata.getDefaultInstance()); + + persistIssuesStep = new PersistIssuesStep(dbClient, System2.INSTANCE, new UpdateConflictResolver(), new RuleRepositoryImpl(mock(AdHocRuleCreator.class), dbClient), + periodHolder, protoIssueCache, new IssueStorage(), UuidFactoryImpl.INSTANCE); + + project = db.components().insertPrivateProject().getMainBranchComponent(); + scaRule = db.rules().insert(RuleTesting.newRule(RuleKey.of("external_sca", "use-of-vulnerable-dependency"))); + cve = new CveDto("cve_uuid", "CVE-123", "Some CVE description", 1.0, 2.0, 3.0, 4L, 5L, 6L, 7L); + dbClient.cveDao().insert(dbSession, cve); + db.commit(); + } + + @Test + void execute_shouldInsertCveId_whenNewCveIssue() { + String issueKey = "ISSUE_KEY"; + DefaultIssue defaultIssue = new DefaultIssue() + .setCveId(cve.id()) + .setKey(issueKey) + .setType(VULNERABILITY) + .setComponentKey(project.getKey()) + .setComponentUuid(project.uuid()) + .setProjectUuid(project.uuid()) + .setProjectKey(project.getKey()) + .setRuleKey(scaRule.getKey()) + .setStatus(OPEN.toString()) + .setCreationDate(new Date()); + protoIssueCache.newAppender().append(defaultIssue).close(); + + Context context = new TestComputationStepContext(); + persistIssuesStep.execute(context); + + List> select = db.select(db.getSession(), "select * from issues_dependency"); + Assertions.assertThat(select).hasSize(1); + Assertions.assertThat(select.get(0)).containsExactlyInAnyOrderEntriesOf( + Map.of( + "issue_uuid", issueKey, + "cve_uuid", cve.uuid()) + ); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReader.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReader.java index f92f857574d..de85e094a67 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReader.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReader.java @@ -70,4 +70,6 @@ public interface BatchReportReader { Optional readComponentChangedLines(int fileRef); CloseableIterator readAnalysisWarnings(); + + CloseableIterator readCves(); } diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderImpl.java index a7ce30c32fe..df17a5ac89b 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderImpl.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderImpl.java @@ -227,4 +227,10 @@ public class BatchReportReaderImpl implements BatchReportReader { ensureInitialized(); return delegate.readAnalysisWarnings(); } + + @Override + public CloseableIterator readCves() { + ensureInitialized(); + return delegate.readCves(); + } } diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/TrackerRawInputFactory.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/TrackerRawInputFactory.java index 3100a1368b6..93b6054564e 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/TrackerRawInputFactory.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/TrackerRawInputFactory.java @@ -270,6 +270,7 @@ public class TrackerRawInputFactory { } issue.setIsFromExternalRuleEngine(true); issue.setLocations(dbLocationsBuilder.build()); + issue.setCveId(reportExternalIssue.getCveId()); return issue; } diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistCveStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistCveStep.java new file mode 100644 index 00000000000..34bd2ad9a3f --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistCveStep.java @@ -0,0 +1,139 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.step; + +import java.util.HashSet; +import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.utils.System2; +import org.sonar.ce.task.projectanalysis.batch.BatchReportReader; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.core.util.CloseableIterator; +import org.sonar.core.util.UuidFactory; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.dependency.CveCweDto; +import org.sonar.db.dependency.CveDto; +import org.sonar.scanner.protocol.output.ScannerReport; + +/** + * Step that persists CVEs and their CWEs in the database. + * CVEs are inserted or updated in the database based on the information from the scanner report. + * If CWEs need to be updated, we simply remove all CWEs from the CVE and insert what is sent by the scanner. + */ +public class PersistCveStep implements ComputationStep { + + private static final Logger LOG = LoggerFactory.getLogger(PersistCveStep.class); + + private final BatchReportReader batchReportReader; + private final DbClient dbClient; + private final UuidFactory uuidFactory; + private final System2 system2; + + public PersistCveStep(BatchReportReader batchReportReader, DbClient dbClient, UuidFactory uuidFactory, System2 system2) { + this.batchReportReader = batchReportReader; + this.dbClient = dbClient; + this.uuidFactory = uuidFactory; + this.system2 = system2; + } + + @Override + public String getDescription() { + return "Persist CVEs"; + } + + @Override + public void execute(Context context) { + int count = 0; + try (DbSession dbSession = dbClient.openSession(false); + CloseableIterator batchCves = batchReportReader.readCves()) { + while (batchCves.hasNext()) { + updateOrInsertCve(dbSession, batchCves.next()); + count++; + } + LOG.debug("{} CVEs were imported/updated", count); + dbSession.commit(); + } catch (Exception exception) { + throw new IllegalStateException(String.format("CVEs import failed after processing %d CVEs successfully", count), exception); + } + } + + private void updateOrInsertCve(DbSession dbSession, ScannerReport.Cve scannerCve) { + dbClient.cveDao().selectById(dbSession, scannerCve.getCveId()) + .ifPresentOrElse( + cveDto -> updateCve(dbSession, cveDto, scannerCve), + () -> insertCve(dbSession, scannerCve)); + } + + private void updateCve(DbSession dbSession, CveDto cveInDb, ScannerReport.Cve scannerCve) { + CveDto dtoForUpdate = toDtoForUpdate(scannerCve, cveInDb); + dbClient.cveDao().update(dbSession, dtoForUpdate); + String cveUuid = cveInDb.uuid(); + deleteThenInsertCwesIfUpdated(dbSession, scannerCve, cveUuid); + } + + private CveDto toDtoForUpdate(ScannerReport.Cve cve, CveDto cveInDb) { + return new CveDto( + cveInDb.uuid(), + cve.getCveId(), + cve.getDescription(), + cve.getCvssScore(), + cve.getEpssScore(), + cve.getEpssPercentile(), + cve.getPublishedDate(), + cve.getLastModifiedDate(), + cveInDb.createdAt(), + system2.now() + ); + } + + private void deleteThenInsertCwesIfUpdated(DbSession dbSession, ScannerReport.Cve scannerCve, String cveUuid) { + Set cweInDb = dbClient.cveCweDao().selectByCveUuid(dbSession, cveUuid); + Set cweFromReport = new HashSet<>(scannerCve.getCweList()); + if (!cweInDb.equals(cweFromReport)) { + dbClient.cveCweDao().deleteByCveUuid(dbSession, cveUuid); + cweFromReport.forEach(cwe -> dbClient.cveCweDao().insert(dbSession, new CveCweDto(cveUuid, cwe))); + } + } + + private void insertCve(DbSession dbSession, ScannerReport.Cve scannerCve) { + CveDto dtoForInsert = toDtoForInsert(scannerCve); + dbClient.cveDao().insert(dbSession, dtoForInsert); + scannerCve.getCweList().forEach(cwe -> dbClient.cveCweDao().insert(dbSession, new CveCweDto(dtoForInsert.uuid(), cwe))); + } + + private CveDto toDtoForInsert(ScannerReport.Cve cve) { + long now = system2.now(); + return new CveDto( + uuidFactory.create(), + cve.getCveId(), + cve.getDescription(), + cve.getCvssScore(), + cve.getEpssScore(), + cve.getEpssPercentile(), + cve.getPublishedDate(), + cve.getLastModifiedDate(), + now, + now + ); + } + +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistIssuesStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistIssuesStep.java index a551254d0b4..1f6aebe48b8 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistIssuesStep.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistIssuesStep.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; import org.sonar.api.utils.System2; @@ -37,6 +38,8 @@ import org.sonar.core.util.UuidFactory; import org.sonar.db.BatchSession; import org.sonar.db.DbClient; import org.sonar.db.DbSession; +import org.sonar.db.dependency.CveDto; +import org.sonar.db.dependency.IssuesDependencyDto; import org.sonar.db.issue.AnticipatedTransitionMapper; import org.sonar.db.issue.IssueChangeMapper; import org.sonar.db.issue.IssueDao; @@ -46,6 +49,7 @@ import org.sonar.db.newcodeperiod.NewCodePeriodType; import org.sonar.server.issue.IssueStorage; import static org.sonar.core.util.FileUtils.humanReadableByteCountSI; +import static org.sonar.db.issue.IssueDto.toDtoForComputationInsert; public class PersistIssuesStep implements ComputationStep { // holding up to 1000 DefaultIssue (max size of addedIssues and updatedIssues at any given time) in memory should not @@ -134,7 +138,7 @@ public class PersistIssuesStep implements ComputationStep { List issueDtos = new LinkedList<>(); addedIssues.forEach(addedIssue -> { String ruleUuid = ruleRepository.getByKey(addedIssue.ruleKey()).getUuid(); - IssueDto dto = IssueDto.toDtoForComputationInsert(addedIssue, ruleUuid, now); + IssueDto dto = toDtoForComputationInsert(addedIssue, ruleUuid, now); issueDao.insertWithoutImpacts(dbSession, dto); issueDtos.add(dto); if (isOnBranchUsingReferenceBranch() && addedIssue.isOnChangedLine()) { @@ -145,7 +149,15 @@ public class PersistIssuesStep implements ComputationStep { addedIssue.getAnticipatedTransitionUuid().ifPresent(anticipatedTransitionMapper::delete); }); - issueDtos.forEach(issueDto -> issueDao.insertIssueImpacts(dbSession, issueDto)); + issueDtos.forEach(issueDto -> insertAdditionalIssueData(issueDao, dbSession, issueDto)); + } + + private void insertAdditionalIssueData(IssueDao issueDao, DbSession dbSession, IssueDto issueDto) { + issueDao.insertIssueImpacts(dbSession, issueDto); + if (issueDto.getCveId() != null) { + Consumer insertIssueDependency = cveDto -> dbClient.issuesDependencyDao().insert(dbSession, new IssuesDependencyDto(issueDto.getKey(), cveDto.uuid())); + dbClient.cveDao().selectById(dbSession, issueDto.getCveId()).ifPresent(insertIssueDependency); + } } private void persistUpdatedIssues(IssueStatistics statistics, List updatedIssues, IssueDao issueDao, diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java index 9fbf27e0a35..34df5531d9d 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java @@ -109,6 +109,7 @@ public class ReportComputationSteps extends AbstractComputationSteps { PersistLiveMeasuresStep.class, PersistDuplicationDataStep.class, PersistAdHocRulesStep.class, + PersistCveStep.class, PersistIssuesStep.class, CleanIssueChangesStep.class, PersistProjectLinksStep.class, diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/util/cache/ProtobufIssueDiskCache.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/util/cache/ProtobufIssueDiskCache.java index c3d9d139609..5b2760f1750 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/util/cache/ProtobufIssueDiskCache.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/util/cache/ProtobufIssueDiskCache.java @@ -151,6 +151,7 @@ public class ProtobufIssueDiskCache implements DiskCache { for (IssueCache.FieldDiffs protoFieldDiffs : next.getChangesList()) { defaultIssue.addChange(toDefaultIssueChanges(protoFieldDiffs)); } + defaultIssue.setCveId(next.hasCveId() ? next.getCveId() : null); return defaultIssue; } @@ -213,7 +214,7 @@ public class ProtobufIssueDiskCache implements DiskCache { for (FieldDiffs fieldDiffs : defaultIssue.changes()) { builder.addChanges(toProtoIssueChanges(fieldDiffs)); } - + ofNullable(defaultIssue.getCveId()).ifPresent(builder::setCveId); return builder.build(); } diff --git a/server/sonar-ce-task-projectanalysis/src/main/protobuf/issue_cache.proto b/server/sonar-ce-task-projectanalysis/src/main/protobuf/issue_cache.proto index acd3e4e62c3..8ef383a8687 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/protobuf/issue_cache.proto +++ b/server/sonar-ce-task-projectanalysis/src/main/protobuf/issue_cache.proto @@ -30,85 +30,86 @@ option optimize_for = SPEED; message Issue { optional string key = 1; - optional int32 ruleType = 2; - optional string componentUuid = 3; - optional string componentKey = 4; - reserved 5; //moduleUuid - reserved 6; // moduleUuidPath - optional string projectUuid = 7; - optional string projectKey = 8; - optional string ruleKey = 9; + optional int32 rule_type = 2; + optional string component_uuid = 3; + optional string component_key = 4; + reserved 5; //module_uuid + reserved 6; // module_uuid_path + optional string project_uuid = 7; + optional string project_key = 8; + optional string rule_key = 9; optional string language = 10; optional string severity = 11; - optional bool manualSeverity = 12; + optional bool manual_severity = 12; optional string message = 13; optional int32 line = 14; optional double gap = 15; optional int64 effort = 16; optional string status = 17; optional string resolution = 18; - optional string assigneeUuid = 19; + optional string assignee_uuid = 19; optional string checksum = 20; - optional string authorLogin = 22; + optional string author_login = 22; optional string tags = 23; optional sonarqube.db.issues.Locations locations = 24; - optional bool isFromExternalRuleEngine = 25; + optional bool is_from_external_rule_engine = 25; // FUNCTIONAL DATES - optional int64 creationDate = 26; - optional int64 updateDate = 27; - optional int64 closeDate = 28; + optional int64 creation_date = 26; + optional int64 update_date = 27; + optional int64 close_date = 28; repeated FieldDiffs changes = 29; - optional FieldDiffs currentChanges = 30; + optional FieldDiffs current_changes = 30; - optional bool isNew = 31; - optional bool isCopied = 32; - optional bool beingClosed = 33; - optional bool onDisabledRule = 34; - optional bool isChanged = 35; - optional bool sendNotifications = 36; - optional int64 selectedAt = 37; + optional bool is_new = 31; + optional bool is_copied = 32; + optional bool being_closed = 33; + optional bool on_disabled_rule = 34; + optional bool is_changed = 35; + optional bool send_notifications = 36; + optional int64 selected_at = 37; repeated Comment comments = 38; - optional bool quickFixAvailable = 39; - - optional bool isOnReferencedBranch = 40; - optional bool isOnChangedLine = 41; - optional bool isNewCodeReferenceIssue = 42; - optional bool isNoLongerNewCodeReferenceIssue = 43; - optional string ruleDescriptionContextKey = 44; - optional sonarqube.db.issues.MessageFormattings messageFormattings = 45; - optional string codeVariants = 46; - optional string assigneeLogin = 47; - optional string anticipatedTransitionUuid = 48; + optional bool quick_fix_available = 39; + + optional bool is_on_referenced_branch = 40; + optional bool is_on_changed_line = 41; + optional bool is_new_code_reference_issue = 42; + optional bool is_no_longer_new_code_reference_issue = 43; + optional string rule_description_context_key = 44; + optional sonarqube.db.issues.MessageFormattings message_formattings = 45; + optional string code_variants = 46; + optional string assignee_login = 47; + optional string anticipated_transition_uuid = 48; repeated Impact impacts = 49; - optional string cleanCodeAttribute = 50; - optional bool isPrioritizedRule = 51; + optional string clean_code_attribute = 50; + optional bool is_prioritized_rule = 51; + optional string cve_id = 52; } message Comment { - optional string issueKey = 1; - optional string userUuid = 2; - optional int64 createdAt = 3; - optional int64 updatedAt = 4; + optional string issue_key = 1; + optional string user_uuid = 2; + optional int64 created_at = 3; + optional int64 updated_at = 4; optional string key = 5; - optional string markdownText = 6; - optional bool isNew = 7; + optional string markdown_text = 6; + optional bool is_new = 7; } message FieldDiffs { - optional string userUuid = 1; - optional int64 creationDate = 2; - optional string issueKey = 3; + optional string user_uuid = 1; + optional int64 creation_date = 2; + optional string issue_key = 3; map diffs = 4; } message Diff { - optional string oldValue = 1; - optional string newValue = 2; + optional string old_value = 1; + optional string new_value = 2; } message Impact { diff --git a/server/sonar-ce-task-projectanalysis/src/main/protobuf/project_dump.proto b/server/sonar-ce-task-projectanalysis/src/main/protobuf/project_dump.proto index 1d99ac0ceba..d1e3782b7f1 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/protobuf/project_dump.proto +++ b/server/sonar-ce-task-projectanalysis/src/main/protobuf/project_dump.proto @@ -53,12 +53,12 @@ message Branch { message Analysis { int64 component_ref = 2; int64 date = 3; - string projectVersion = 4; + string project_version = 4; string period1_mode = 5; string period1_param = 6; int64 period1_date = 7; string uuid = 8; - string buildString = 9; + string build_string = 9; } message Metric { @@ -123,10 +123,10 @@ message Issue { string project_uuid = 22; string rule_ref = 23; - optional string ruleDescriptionContextKey = 24; + optional string rule_description_context_key = 24; - repeated MessageFormatting messageFormattings = 25; - string codeVariants = 26; + repeated MessageFormatting message_formattings = 25; + string code_variants = 26; repeated Impact impacts = 27; } @@ -177,7 +177,7 @@ message AdHocRule { string scope = 8; RuleMetadata metadata = 9; repeated Impact impacts = 11; - optional string cleanCodeAttribute = 12; + optional string clean_code_attribute = 12; } // stream of messages in settings.pb @@ -253,4 +253,4 @@ enum Severity { LOW = 0; MEDIUM = 1; HIGH = 2; -} \ No newline at end of file +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderImplTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderImplTest.java index d181d85263f..7019e756d2a 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderImplTest.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderImplTest.java @@ -30,6 +30,7 @@ import org.sonar.api.impl.utils.JUnitTempFolder; import org.sonar.core.util.CloseableIterator; import org.sonar.scanner.protocol.output.FileStructure; import org.sonar.scanner.protocol.output.ScannerReport; +import org.sonar.scanner.protocol.output.ScannerReport.Cve; import org.sonar.scanner.protocol.output.ScannerReportWriter; import static com.google.common.collect.ImmutableList.of; @@ -300,4 +301,28 @@ public class BatchReportReaderImplTest { assertThat(res).toIterable().containsExactlyElementsOf(warnings); res.close(); } + + @Test + public void readCves_shouldReturnCves() { + Cve cve1 = builCve("1").build(); + writer.appendCve(cve1); + Cve cve2 = builCve("2").build(); + writer.appendCve(cve2); + + CloseableIterator cveCloseableIterator = underTest.readCves(); + + assertThat(cveCloseableIterator).toIterable().containsExactlyInAnyOrder(cve1, cve2); + } + + private Cve.Builder builCve(String suffix) { + return Cve.newBuilder() + .setCveId("CVE-" + suffix) + .addCwe("CWE-" + suffix) + .setDescription("Some CVE description " + suffix) + .setCvssScore(7.5F) + .setEpssScore(0.1F) + .setEpssPercentile(0.1F) + .setLastModifiedDate(1L) + .setPublishedDate(2L); + } } diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderRule.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderRule.java index c6075de139a..b960803b18a 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderRule.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderRule.java @@ -62,6 +62,7 @@ public class BatchReportReaderRule implements TestRule, BatchReportReader, After private Map changedLines = new HashMap<>(); private List analysisWarnings = Collections.emptyList(); private byte[] analysisCache; + private List cves = new ArrayList<>(); @Override public Statement apply(final Statement statement, Description description) { @@ -326,4 +327,14 @@ public class BatchReportReaderRule implements TestRule, BatchReportReader, After public void afterEach(ExtensionContext context) { clear(); } + + @Override + public CloseableIterator readCves() { + return CloseableIterator.from(cves.iterator()); + } + + public BatchReportReaderRule putCves(List cves) { + this.cves = cves; + return this; + } } diff --git a/server/sonar-db-dao/src/it/java/org/sonar/db/dependency/CveCweDaoIT.java b/server/sonar-db-dao/src/it/java/org/sonar/db/dependency/CveCweDaoIT.java index 7b9587201f5..ffffd54664a 100644 --- a/server/sonar-db-dao/src/it/java/org/sonar/db/dependency/CveCweDaoIT.java +++ b/server/sonar-db-dao/src/it/java/org/sonar/db/dependency/CveCweDaoIT.java @@ -70,4 +70,23 @@ class CveCweDaoIT { assertThat(result).isEmpty(); } + @Test + void deleteByCveUuid_shouldDeleteCwesAttachedToCve() { + String cveUuid = "CVE_UUID"; + cveCweDao.insert(db.getSession(), new CveCweDto(cveUuid, "CWE-123")); + cveCweDao.insert(db.getSession(), new CveCweDto(cveUuid, "CWE-456")); + CveCweDto nonDeletedCwe = new CveCweDto("ANOTHER_CVE_UUID", "CWE-789"); + cveCweDao.insert(db.getSession(), nonDeletedCwe); + + cveCweDao.deleteByCveUuid(db.getSession(), cveUuid); + + List> result = db.select(db.getSession(), "select * from cve_cwe"); + assertThat(result).hasSize(1); + assertThat(result.get(0)).containsExactlyInAnyOrderEntriesOf( + Map.of( + "cve_uuid", nonDeletedCwe.cveUuid(), + "cwe", nonDeletedCwe.cwe()) + ); + } + } diff --git a/server/sonar-db-dao/src/it/java/org/sonar/db/dependency/CveDaoIT.java b/server/sonar-db-dao/src/it/java/org/sonar/db/dependency/CveDaoIT.java index 00c04e9df21..79bc9530ae0 100644 --- a/server/sonar-db-dao/src/it/java/org/sonar/db/dependency/CveDaoIT.java +++ b/server/sonar-db-dao/src/it/java/org/sonar/db/dependency/CveDaoIT.java @@ -89,4 +89,17 @@ class CveDaoIT { assertThat(result).usingRecursiveComparison().isEqualTo(cveDto); } + @Test + void update_shouldUpdateCveButCreationDate() { + CveDto insertedDto = new CveDto("uuid-1", "CVE-1", "Some CVE description 1", 1.0, 2.0, 3.0, 4L, 5L, 6L, 7L); + cveDao.insert(db.getSession(), insertedDto); + CveDto updatedDto = new CveDto("uuid-1", "CVE-2", "Some CVE description 2", 7.0, 1.0, 2.0, 3L, 4L, 5L, 6L); + + cveDao.update(db.getSession(), updatedDto); + + CveDto result = cveDao.selectById(db.getSession(), updatedDto.id()).orElseGet(() -> fail("CVE not found in database")); + assertThat(result).usingRecursiveComparison().ignoringFields("createdAt").isEqualTo(updatedDto); + assertThat(result.createdAt()).isEqualTo(insertedDto.createdAt()); + } + } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/dependency/CveCweDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/dependency/CveCweDao.java index c974ae08e9f..df6b5e5fed4 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/dependency/CveCweDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/dependency/CveCweDao.java @@ -36,4 +36,9 @@ public class CveCweDao implements Dao { public Set selectByCveUuid(DbSession dbSession, String cveUuid) { return mapper(dbSession).selectByCveUuid(cveUuid); } + + public void deleteByCveUuid(DbSession dbSession, String cveUuid) { + mapper(dbSession).deleteByCveUuid(cveUuid); + } + } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/dependency/CveCweMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/dependency/CveCweMapper.java index 612eb7ba67f..886a690cd68 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/dependency/CveCweMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/dependency/CveCweMapper.java @@ -25,4 +25,6 @@ public interface CveCweMapper { void insert(CveCweDto cveCweDto); Set selectByCveUuid(String cveUuid); + + void deleteByCveUuid(String cveUuid); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/dependency/CveDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/dependency/CveDao.java index 640ed6eb173..d7201f8ee27 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/dependency/CveDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/dependency/CveDao.java @@ -33,6 +33,10 @@ public class CveDao implements Dao { return Optional.ofNullable(mapper(dbSession).selectById(id)); } + public void update(DbSession dbSession, CveDto cveDto) { + mapper(dbSession).update(cveDto); + } + private static CveMapper mapper(DbSession dbSession) { return dbSession.getMapper(CveMapper.class); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/dependency/CveMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/dependency/CveMapper.java index 1cf344709ce..f4ea3c5325b 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/dependency/CveMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/dependency/CveMapper.java @@ -23,4 +23,6 @@ public interface CveMapper { void insert(CveDto cveDto); CveDto selectById(String id); + + void update(CveDto cveDto); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/issue/IssueDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/issue/IssueDto.java index c60fdb3075d..0107b8c399c 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/issue/IssueDto.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/issue/IssueDto.java @@ -166,7 +166,8 @@ public final class IssueDto implements Serializable { .setCleanCodeAttribute(issue.getCleanCodeAttribute()) // technical dates .setCreatedAt(now) - .setUpdatedAt(now); + .setUpdatedAt(now) + .setCveId(issue.getCveId()); } @NotNull @@ -880,8 +881,9 @@ public final class IssueDto implements Serializable { return cveId; } - public void setCveId(String cveId) { + public IssueDto setCveId(@Nullable String cveId) { this.cveId = cveId; + return this; } @Override diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/dependency/CveCweMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/dependency/CveCweMapper.xml index 5002efd6e28..b335eac5731 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/dependency/CveCweMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/dependency/CveCweMapper.xml @@ -18,4 +18,9 @@ where cve_uuid = #{cveUuid,jdbcType=VARCHAR} + + delete from cve_cwe + where cve_uuid = #{cveUuid,jdbcType=VARCHAR} + + diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/dependency/CveMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/dependency/CveMapper.xml index 22833f96b4f..83d45d500c8 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/dependency/CveMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/dependency/CveMapper.xml @@ -15,7 +15,7 @@ c.updated_at as updatedAt - + insert into cves ( uuid, id, @@ -41,7 +41,7 @@ ) - select from cves c @@ -49,4 +49,19 @@ c.id = #{id, jdbcType=VARCHAR} + + update cves + set + id = #{id, jdbcType=VARCHAR}, + description = #{description, jdbcType=VARCHAR}, + cvss_score = #{cvssScore, jdbcType=DOUBLE}, + epss_score = #{epssScore, jdbcType=DOUBLE}, + epss_percentile = #{epssPercentile, jdbcType=DOUBLE}, + published_at = #{publishedAt, jdbcType=BIGINT}, + last_modified_at = #{lastModifiedAt, jdbcType=BIGINT}, + updated_at = #{updatedAt, jdbcType=BIGINT} + where + uuid = #{uuid, jdbcType=BIGINT} + + diff --git a/sonar-core/src/main/java/org/sonar/core/issue/DefaultIssue.java b/sonar-core/src/main/java/org/sonar/core/issue/DefaultIssue.java index 83427a1a193..7480488b75e 100644 --- a/sonar-core/src/main/java/org/sonar/core/issue/DefaultIssue.java +++ b/sonar-core/src/main/java/org/sonar/core/issue/DefaultIssue.java @@ -140,6 +140,8 @@ public class DefaultIssue implements Issue, Trackable, org.sonar.api.ce.measure. private Map impacts = new EnumMap<>(SoftwareQuality.class); private CleanCodeAttribute cleanCodeAttribute = null; + private String cveId = null; + @Override public String key() { return key; @@ -771,4 +773,13 @@ public class DefaultIssue implements Issue, Trackable, org.sonar.api.ce.measure. public Date getUpdateDate() { return updateDate; } + + public String getCveId() { + return cveId; + } + + public DefaultIssue setCveId(@Nullable String cveId) { + this.cveId = cveId; + return this; + } } -- 2.39.5