From f20a4314ac8462c3c1c28947814775e0154f8c54 Mon Sep 17 00:00:00 2001 From: Jacek Date: Fri, 15 Jul 2022 09:33:39 +0200 Subject: [PATCH] SONAR-16374 Add scheduled task to read PushEvents from DB --- .../pushevent/TaintVulnerabilityVisitor.java | 5 +- .../step/PersistPushEventsStep.java | 1 + .../org/sonar/db/pushevent/PushEventDao.java | 5 + .../org/sonar/db/pushevent/PushEventDto.java | 10 + .../sonar/db/pushevent/PushEventMapper.java | 4 + .../sonar/db/pushevent/PushEventMapper.xml | 44 ++++ server/sonar-db-dao/src/schema/schema-sq.ddl | 1 + .../sonar/db/pushevent/PushEventDaoTest.java | 62 ++++++ .../version/v96/CreatePushEventsTable.java | 1 + ...ushWsModule.java => ServerPushModule.java} | 6 +- .../polling/PushEventExecutorService.java | 28 +++ .../PushEventPollExecutorServiceImpl.java | 44 ++++ .../polling/PushEventPollScheduler.java | 150 +++++++++++++ .../sonarlint/SonarLintClientsRegistry.java | 10 + .../pushapi/ServerPushWsModuleTest.java | 2 +- .../PushEventPollExecutorServiceImplTest.java | 39 ++++ .../polling/PushEventPollSchedulerTest.java | 209 ++++++++++++++++++ .../SonarLintClientsRegistryTest.java | 2 + .../platformlevel/PlatformLevel4.java | 4 +- 19 files changed, 620 insertions(+), 7 deletions(-) rename server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/{ServerPushWsModule.java => ServerPushModule.java} (85%) create mode 100644 server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/scheduler/polling/PushEventExecutorService.java create mode 100644 server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/scheduler/polling/PushEventPollExecutorServiceImpl.java create mode 100644 server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/scheduler/polling/PushEventPollScheduler.java create mode 100644 server/sonar-webserver-pushapi/src/test/java/org/sonar/server/pushapi/scheduler/polling/PushEventPollExecutorServiceImplTest.java create mode 100644 server/sonar-webserver-pushapi/src/test/java/org/sonar/server/pushapi/scheduler/polling/PushEventPollSchedulerTest.java diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/pushevent/TaintVulnerabilityVisitor.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/pushevent/TaintVulnerabilityVisitor.java index e6531e9164e..d64e8925b66 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/pushevent/TaintVulnerabilityVisitor.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/pushevent/TaintVulnerabilityVisitor.java @@ -21,7 +21,6 @@ package org.sonar.ce.task.projectanalysis.pushevent; import java.util.LinkedList; import java.util.List; -import java.util.Set; import org.jetbrains.annotations.NotNull; import org.sonar.api.rules.RuleType; import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolder; @@ -31,12 +30,12 @@ import org.sonar.ce.task.projectanalysis.issue.IssueVisitor; import org.sonar.core.issue.DefaultIssue; import org.sonar.db.protobuf.DbCommons; import org.sonar.db.protobuf.DbIssues; +import org.sonar.server.issue.TaintChecker; import static java.util.Objects.requireNonNull; import static java.util.Objects.requireNonNullElse; public class TaintVulnerabilityVisitor extends IssueVisitor { - private static final Set TAINT_REPOSITORIES = Set.of("roslyn.sonaranalyzer.security.cs", "javasecurity", "jssecurity", "tssecurity", "phpsecurity", "pythonsecurity"); private final PushEventRepository pushEventRepository; private final AnalysisMetadataHolder analysisMetadataHolder; @@ -118,7 +117,7 @@ public class TaintVulnerabilityVisitor extends IssueVisitor { } private static boolean isTaintVulnerability(DefaultIssue issue) { - return TAINT_REPOSITORIES.contains(issue.getRuleKey().repository()) + return TaintChecker.getTaintRepositories().contains(issue.getRuleKey().repository()) && issue.getLocations() != null && !RuleType.SECURITY_HOTSPOT.equals(issue.type()); } diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistPushEventsStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistPushEventsStep.java index 97ab977542a..0d30986a9f2 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistPushEventsStep.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistPushEventsStep.java @@ -73,6 +73,7 @@ public class PersistPushEventsStep implements ComputationStep { private void pushEvent(DbSession dbSession, PushEvent event) { PushEventDto eventDto = new PushEventDto() + .setName(event.getName()) .setProjectUuid(treeRootHolder.getRoot().getUuid()) .setPayload(serializeIssueToPushEvent(event)); dbClient.pushEventDao().insert(dbSession, eventDto); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventDao.java index e8db98fd74f..9adee2badcc 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventDao.java @@ -19,6 +19,7 @@ */ package org.sonar.db.pushevent; +import java.util.Deque; import java.util.Set; import org.sonar.api.utils.System2; import org.sonar.core.util.UuidFactory; @@ -67,4 +68,8 @@ public class PushEventDao implements Dao { return session.getMapper(PushEventMapper.class); } + public Deque selectChunkByProjectUuids(DbSession dbSession, Set projectUuids, + Long lastPullTimestamp, String lastSeenUuid, long count) { + return mapper(dbSession).selectChunkByProjectUuids(projectUuids, lastPullTimestamp, lastSeenUuid, count); + } } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventDto.java index 13f867a8a27..905eaf03b55 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventDto.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventDto.java @@ -23,6 +23,7 @@ import javax.annotation.CheckForNull; public class PushEventDto { private String uuid; + private String name; private String projectUuid; private byte[] payload; private Long createdAt; @@ -40,6 +41,15 @@ public class PushEventDto { return this; } + public String getName() { + return name; + } + + public PushEventDto setName(String name) { + this.name = name; + return this; + } + public String getProjectUuid() { return projectUuid; } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventMapper.java index 6d9ae0fe4e0..78b19c4b4e3 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventMapper.java @@ -19,6 +19,7 @@ */ package org.sonar.db.pushevent; +import java.util.LinkedList; import java.util.List; import java.util.Set; import javax.annotation.CheckForNull; @@ -35,4 +36,7 @@ public interface PushEventMapper { void deleteByUuids(@Param("pushEventUuids") List pushEventUuids); + LinkedList selectChunkByProjectUuids(@Param("projectUuids") Set projectUuids, + @Param("lastPullTimestamp") Long lastPullTimestamp, + @Param("lastSeenUuid") String lastSeenUuid, @Param("count") long count); } diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/pushevent/PushEventMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/pushevent/PushEventMapper.xml index 0f395606bd6..32af916bf01 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/pushevent/PushEventMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/pushevent/PushEventMapper.xml @@ -4,6 +4,7 @@ pe.uuid as uuid, + pe.name as name, pe.project_uuid as projectUuid, pe.payload as payload, pe.created_at as createdAt @@ -12,12 +13,14 @@ INSERT INTO push_events ( uuid, + name, project_uuid, payload, created_at ) VALUES ( #{uuid,jdbcType=VARCHAR}, + #{name,jdbcType=VARCHAR}, #{projectUuid,jdbcType=VARCHAR}, #{payload,jdbcType=BLOB}, #{createdAt,jdbcType=BIGINT} @@ -32,6 +35,47 @@ pe.uuid=#{uuid,jdbcType=VARCHAR} + + + + + +