diff options
19 files changed, 620 insertions, 7 deletions
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/pushevent/TaintVulnerabilityVisitor.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/pushevent/TaintVulnerabilityVisitor.java index e6531e9164e..d64e8925b66 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/pushevent/TaintVulnerabilityVisitor.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/pushevent/TaintVulnerabilityVisitor.java @@ -21,7 +21,6 @@ package org.sonar.ce.task.projectanalysis.pushevent; import java.util.LinkedList; import java.util.List; -import java.util.Set; import org.jetbrains.annotations.NotNull; import org.sonar.api.rules.RuleType; import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolder; @@ -31,12 +30,12 @@ import org.sonar.ce.task.projectanalysis.issue.IssueVisitor; import org.sonar.core.issue.DefaultIssue; import org.sonar.db.protobuf.DbCommons; import org.sonar.db.protobuf.DbIssues; +import org.sonar.server.issue.TaintChecker; import static java.util.Objects.requireNonNull; import static java.util.Objects.requireNonNullElse; public class TaintVulnerabilityVisitor extends IssueVisitor { - private static final Set<String> TAINT_REPOSITORIES = Set.of("roslyn.sonaranalyzer.security.cs", "javasecurity", "jssecurity", "tssecurity", "phpsecurity", "pythonsecurity"); private final PushEventRepository pushEventRepository; private final AnalysisMetadataHolder analysisMetadataHolder; @@ -118,7 +117,7 @@ public class TaintVulnerabilityVisitor extends IssueVisitor { } private static boolean isTaintVulnerability(DefaultIssue issue) { - return TAINT_REPOSITORIES.contains(issue.getRuleKey().repository()) + return TaintChecker.getTaintRepositories().contains(issue.getRuleKey().repository()) && issue.getLocations() != null && !RuleType.SECURITY_HOTSPOT.equals(issue.type()); } diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistPushEventsStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistPushEventsStep.java index 97ab977542a..0d30986a9f2 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistPushEventsStep.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistPushEventsStep.java @@ -73,6 +73,7 @@ public class PersistPushEventsStep implements ComputationStep { private void pushEvent(DbSession dbSession, PushEvent<?> event) { PushEventDto eventDto = new PushEventDto() + .setName(event.getName()) .setProjectUuid(treeRootHolder.getRoot().getUuid()) .setPayload(serializeIssueToPushEvent(event)); dbClient.pushEventDao().insert(dbSession, eventDto); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventDao.java index e8db98fd74f..9adee2badcc 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventDao.java @@ -19,6 +19,7 @@ */ package org.sonar.db.pushevent; +import java.util.Deque; import java.util.Set; import org.sonar.api.utils.System2; import org.sonar.core.util.UuidFactory; @@ -67,4 +68,8 @@ public class PushEventDao implements Dao { return session.getMapper(PushEventMapper.class); } + public Deque<PushEventDto> selectChunkByProjectUuids(DbSession dbSession, Set<String> projectUuids, + Long lastPullTimestamp, String lastSeenUuid, long count) { + return mapper(dbSession).selectChunkByProjectUuids(projectUuids, lastPullTimestamp, lastSeenUuid, count); + } } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventDto.java index 13f867a8a27..905eaf03b55 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventDto.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventDto.java @@ -23,6 +23,7 @@ import javax.annotation.CheckForNull; public class PushEventDto { private String uuid; + private String name; private String projectUuid; private byte[] payload; private Long createdAt; @@ -40,6 +41,15 @@ public class PushEventDto { return this; } + public String getName() { + return name; + } + + public PushEventDto setName(String name) { + this.name = name; + return this; + } + public String getProjectUuid() { return projectUuid; } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventMapper.java index 6d9ae0fe4e0..78b19c4b4e3 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventMapper.java @@ -19,6 +19,7 @@ */ package org.sonar.db.pushevent; +import java.util.LinkedList; import java.util.List; import java.util.Set; import javax.annotation.CheckForNull; @@ -35,4 +36,7 @@ public interface PushEventMapper { void deleteByUuids(@Param("pushEventUuids") List<String> pushEventUuids); + LinkedList<PushEventDto> selectChunkByProjectUuids(@Param("projectUuids") Set<String> projectUuids, + @Param("lastPullTimestamp") Long lastPullTimestamp, + @Param("lastSeenUuid") String lastSeenUuid, @Param("count") long count); } diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/pushevent/PushEventMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/pushevent/PushEventMapper.xml index 0f395606bd6..32af916bf01 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/pushevent/PushEventMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/pushevent/PushEventMapper.xml @@ -4,6 +4,7 @@ <sql id="pushEventColumns"> pe.uuid as uuid, + pe.name as name, pe.project_uuid as projectUuid, pe.payload as payload, pe.created_at as createdAt @@ -12,12 +13,14 @@ <insert id="insert" parameterType="map" useGeneratedKeys="false"> INSERT INTO push_events ( uuid, + name, project_uuid, payload, created_at ) VALUES ( #{uuid,jdbcType=VARCHAR}, + #{name,jdbcType=VARCHAR}, #{projectUuid,jdbcType=VARCHAR}, #{payload,jdbcType=BLOB}, #{createdAt,jdbcType=BIGINT} @@ -32,6 +35,47 @@ pe.uuid=#{uuid,jdbcType=VARCHAR} </select> + <select id="selectChunkByProjectUuids" parameterType="map" resultType="PushEvent"> + SELECT + <include refid="pushEventColumns"/> + FROM push_events pe + WHERE created_at >= #{lastPullTimestamp,jdbcType=BIGINT} + AND ( created_at > #{lastPullTimestamp,jdbcType=BIGINT} OR uuid > #{lastSeenUuid,jdbcType=VARCHAR} ) + AND pe.project_uuid in + <foreach collection="projectUuids" open="(" close=")" item="uuid" separator=","> + #{uuid,jdbcType=VARCHAR} + </foreach> + ORDER BY created_at, uuid + LIMIT #{count} + </select> + + <select id="selectChunkByProjectUuids" parameterType="map" resultType="PushEvent" databaseId="mssql"> + SELECT top (#{count,jdbcType=BIGINT}) + <include refid="pushEventColumns"/> + FROM push_events pe + WHERE created_at >= #{lastPullTimestamp,jdbcType=BIGINT} + AND ( created_at > #{lastPullTimestamp,jdbcType=BIGINT} OR uuid > #{lastSeenUuid,jdbcType=VARCHAR} ) + AND pe.project_uuid in + <foreach collection="projectUuids" open="(" close=")" item="uuid" separator=","> + #{uuid,jdbcType=VARCHAR} + </foreach> + ORDER BY created_at, uuid + </select> + + <select id="selectChunkByProjectUuids" parameterType="map" resultType="PushEvent" databaseId="oracle"> + SELECT * FROM (SELECT + <include refid="pushEventColumns"/> + FROM push_events pe + WHERE created_at >= #{lastPullTimestamp,jdbcType=BIGINT} + AND ( created_at > #{lastPullTimestamp,jdbcType=BIGINT} OR uuid > #{lastSeenUuid,jdbcType=VARCHAR} ) + AND pe.project_uuid in + <foreach collection="projectUuids" open="(" close=")" item="uuid" separator=","> + #{uuid,jdbcType=VARCHAR} + </foreach> + ORDER BY created_at, uuid) + WHERE rownum <= #{count,jdbcType=BIGINT} + </select> + <select id="selectUuidsOfExpiredEvents" parameterType="long" resultType="string"> SELECT pe.uuid diff --git a/server/sonar-db-dao/src/schema/schema-sq.ddl b/server/sonar-db-dao/src/schema/schema-sq.ddl index 072f04334fa..f2294d241f7 100644 --- a/server/sonar-db-dao/src/schema/schema-sq.ddl +++ b/server/sonar-db-dao/src/schema/schema-sq.ddl @@ -742,6 +742,7 @@ CREATE INDEX "PROPERTIES_KEY" ON "PROPERTIES"("PROP_KEY" NULLS FIRST); CREATE TABLE "PUSH_EVENTS"( "UUID" CHARACTER VARYING(40) NOT NULL, + "NAME" CHARACTER VARYING(40) NOT NULL, "PROJECT_UUID" CHARACTER VARYING(40) NOT NULL, "PAYLOAD" BINARY LARGE OBJECT NOT NULL, "CREATED_AT" BIGINT NOT NULL diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/pushevent/PushEventDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/pushevent/PushEventDaoTest.java index e72ab4d4b28..d19c35aba69 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/pushevent/PushEventDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/pushevent/PushEventDaoTest.java @@ -20,9 +20,11 @@ package org.sonar.db.pushevent; import java.util.Set; +import java.util.stream.IntStream; import org.junit.Rule; import org.junit.Test; import org.sonar.api.impl.utils.TestSystem2; +import org.sonar.core.util.UuidFactoryFast; import org.sonar.db.DbSession; import org.sonar.db.DbTester; @@ -45,11 +47,14 @@ public class PushEventDaoTest { PushEventDto eventDtoFirst = new PushEventDto() .setUuid("test-uuid") + .setName("Event") .setProjectUuid("project-uuid") .setPayload("some-event".getBytes(UTF_8)); PushEventDto eventDtoSecond = new PushEventDto() .setProjectUuid("project-uuid") + .setName("Event") + .setPayload("some-event".getBytes(UTF_8)); underTest.insert(session, eventDtoFirst); @@ -71,16 +76,19 @@ public class PushEventDaoTest { @Test public void select_expired_events() { PushEventDto eventDtoFirst = new PushEventDto() + .setName("Event") .setProjectUuid("project-uuid") .setCreatedAt(1000L) .setPayload("some-event".getBytes(UTF_8)); PushEventDto eventDtoSecond = new PushEventDto() + .setName("Event") .setProjectUuid("project-uuid") .setCreatedAt(1000L) .setPayload("some-event".getBytes(UTF_8)); PushEventDto eventDtoThird = new PushEventDto() + .setName("Event") .setProjectUuid("project-uuid") .setCreatedAt(2000L) .setPayload("some-event".getBytes(UTF_8)); @@ -97,11 +105,13 @@ public class PushEventDaoTest { @Test public void delete_events_in_batches() { PushEventDto eventDtoFirst = new PushEventDto() + .setName("Event") .setProjectUuid("project-uuid") .setCreatedAt(1000L) .setPayload("some-event".getBytes(UTF_8)); PushEventDto eventDtoSecond = new PushEventDto() + .setName("Event") .setProjectUuid("project-uuid") .setCreatedAt(1000L) .setPayload("some-event".getBytes(UTF_8)); @@ -114,4 +124,56 @@ public class PushEventDaoTest { assertThat(underTest.selectUuidsOfExpiredEvents(db.getSession(), 2000L)).isEmpty(); } + @Test + public void selectChunkByProjectKeys() { + system2.setNow(1L); + generatePushEvent("proj1"); + system2.tick(); // tick=2 + generatePushEvent("proj2"); + + system2.tick(); // tick=3 + var eventDto4 = generatePushEvent("proj2"); + + var events = underTest.selectChunkByProjectUuids(session, Set.of("proj1", "proj2"), 2L, null, 10); + + // tick=1 and tick=2 skipped + assertThat(events).extracting(PushEventDto::getUuid).containsExactly(eventDto4.getUuid()); + + system2.tick(); // tick=4 + var eventDto5 = generatePushEvent("proj2"); + var eventDto6 = generatePushEvent("proj2"); + + system2.tick(); // tick =5 + var eventDto7 = generatePushEvent("proj2"); + + events = underTest.selectChunkByProjectUuids(session, Set.of("proj1", "proj2"), eventDto4.getCreatedAt(), eventDto4.getUuid(), 10); + assertThat(events).extracting(PushEventDto::getUuid).containsExactly(eventDto5.getUuid(), eventDto6.getUuid(), eventDto7.getUuid()); + } + + @Test + public void selectChunkByProjectKeys_pagination() { + system2.setNow(3L); + + IntStream.range(1, 10) + .forEach(value -> generatePushEvent("event-" + value, "proj1")); + + var events = underTest.selectChunkByProjectUuids(session, Set.of("proj1"), 1L, null, 3); + assertThat(events).extracting(PushEventDto::getUuid).containsExactly("event-1", "event-2", "event-3"); + + events = underTest.selectChunkByProjectUuids(session, Set.of("proj1"), 3L, "event-3", 3); + assertThat(events).extracting(PushEventDto::getUuid).containsExactly("event-4", "event-5", "event-6"); + } + + private PushEventDto generatePushEvent(String projectUuid) { + return generatePushEvent(UuidFactoryFast.getInstance().create(), projectUuid); + } + + private PushEventDto generatePushEvent(String uuid, String projectUuid) { + return underTest.insert(session, new PushEventDto() + .setName("Event") + .setUuid(uuid) + .setProjectUuid(projectUuid) + .setPayload("some-event".getBytes(UTF_8))); + } + } diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v96/CreatePushEventsTable.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v96/CreatePushEventsTable.java index f44a7b5a3a6..e1d1d8a994a 100644 --- a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v96/CreatePushEventsTable.java +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v96/CreatePushEventsTable.java @@ -41,6 +41,7 @@ public class CreatePushEventsTable extends CreateTableChange { public void execute(Context context, String tableName) throws SQLException { context.execute(new CreateTableBuilder(getDialect(), tableName) .addPkColumn(newVarcharColumnDefBuilder().setColumnName("uuid").setIsNullable(false).setLimit(UUID_SIZE).build()) + .addColumn(newVarcharColumnDefBuilder().setColumnName("name").setIsNullable(false).setLimit(40).build()) .addColumn(newVarcharColumnDefBuilder().setColumnName("project_uuid").setIsNullable(false).setLimit(UUID_SIZE).build()) .addColumn(newBlobColumnDefBuilder().setColumnName("payload").setIsNullable(false).build()) .addColumn(newBigIntegerColumnDefBuilder().setColumnName("created_at").setIsNullable(false).build()) diff --git a/server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/ServerPushWsModule.java b/server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/ServerPushModule.java index 091b56d7d56..f8f248abe6a 100644 --- a/server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/ServerPushWsModule.java +++ b/server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/ServerPushModule.java @@ -20,6 +20,8 @@ package org.sonar.server.pushapi; import org.sonar.core.platform.Module; +import org.sonar.server.pushapi.scheduler.polling.PushEventPollExecutorServiceImpl; +import org.sonar.server.pushapi.scheduler.polling.PushEventPollScheduler; import org.sonar.server.pushapi.scheduler.purge.PushEventsPurgeExecutorServiceImpl; import org.sonar.server.pushapi.scheduler.purge.PushEventsPurgeInitializer; import org.sonar.server.pushapi.scheduler.purge.PushEventsPurgeSchedulerImpl; @@ -27,7 +29,7 @@ import org.sonar.server.pushapi.sonarlint.SonarLintClientPermissionsValidator; import org.sonar.server.pushapi.sonarlint.SonarLintClientsRegistry; import org.sonar.server.pushapi.sonarlint.SonarLintPushAction; -public class ServerPushWsModule extends Module { +public class ServerPushModule extends Module { @Override protected void configureModule() { @@ -37,6 +39,8 @@ public class ServerPushWsModule extends Module { SonarLintClientsRegistry.class, SonarLintPushAction.class, + PushEventPollExecutorServiceImpl.class, + PushEventPollScheduler.class, // Push Events Purge PushEventsPurgeSchedulerImpl.class, PushEventsPurgeExecutorServiceImpl.class, diff --git a/server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/scheduler/polling/PushEventExecutorService.java b/server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/scheduler/polling/PushEventExecutorService.java new file mode 100644 index 00000000000..56a7510e5c6 --- /dev/null +++ b/server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/scheduler/polling/PushEventExecutorService.java @@ -0,0 +1,28 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.pushapi.scheduler.polling; + +import java.util.concurrent.ScheduledExecutorService; +import org.sonar.api.server.ServerSide; + +@ServerSide +public interface PushEventExecutorService extends ScheduledExecutorService { + +} diff --git a/server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/scheduler/polling/PushEventPollExecutorServiceImpl.java b/server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/scheduler/polling/PushEventPollExecutorServiceImpl.java new file mode 100644 index 00000000000..fb356c5e1ca --- /dev/null +++ b/server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/scheduler/polling/PushEventPollExecutorServiceImpl.java @@ -0,0 +1,44 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.pushapi.scheduler.polling; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import org.sonar.api.server.ServerSide; +import org.sonar.server.util.AbstractStoppableScheduledExecutorServiceImpl; + +import static java.lang.Thread.MIN_PRIORITY; +import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; + +public class PushEventPollExecutorServiceImpl extends + AbstractStoppableScheduledExecutorServiceImpl<ScheduledExecutorService> implements PushEventExecutorService { + + public PushEventPollExecutorServiceImpl() { + super(newSingleThreadScheduledExecutor(PushEventPollExecutorServiceImpl::createThread)); + } + + static Thread createThread(Runnable r) { + Thread thread = Executors.defaultThreadFactory().newThread(r); + thread.setName("PushEventPoll-%d"); + thread.setPriority(MIN_PRIORITY); + thread.setDaemon(true); + return thread; + } +} diff --git a/server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/scheduler/polling/PushEventPollScheduler.java b/server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/scheduler/polling/PushEventPollScheduler.java new file mode 100644 index 00000000000..883c943bceb --- /dev/null +++ b/server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/scheduler/polling/PushEventPollScheduler.java @@ -0,0 +1,150 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.pushapi.scheduler.polling; + +import java.util.Collection; +import java.util.Deque; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.jetbrains.annotations.NotNull; +import org.sonar.api.Startable; +import org.sonar.api.config.Configuration; +import org.sonar.api.server.ServerSide; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.project.ProjectDto; +import org.sonar.db.pushevent.PushEventDto; +import org.sonar.server.pushapi.sonarlint.SonarLintClient; +import org.sonar.server.pushapi.sonarlint.SonarLintClientsRegistry; + +@ServerSide +public class PushEventPollScheduler implements Startable { + + private static final Logger LOG = Loggers.get(PushEventPollScheduler.class); + + private static final String INITIAL_DELAY_IN_SECONDS = "sonar.pushevents.polling.initial.delay"; + private static final String PERIOD_IN_SECONDS = "sonar.pushevents.polling.period"; + private static final String PAGE_SIZE = "sonar.pushevents.polling.page.size"; + + private final PushEventExecutorService executorService; + private final SonarLintClientsRegistry clientsRegistry; + private final DbClient dbClient; + private final System2 system2; + private final Configuration config; + private Long lastPullTimestamp = null; + private String lastSeenUuid = null; + + public PushEventPollScheduler(PushEventExecutorService executorService, SonarLintClientsRegistry clientsRegistry, + DbClient dbClient, System2 system2, Configuration config) { + this.executorService = executorService; + this.clientsRegistry = clientsRegistry; + this.dbClient = dbClient; + this.system2 = system2; + this.config = config; + } + + @Override + public void start() { + this.executorService.scheduleAtFixedRate(this::tryBroadcastEvents, getInitialDelay(), getPeriod(), TimeUnit.SECONDS); + } + + private void tryBroadcastEvents() { + try { + doBroadcastEvents(); + } catch (Exception e) { + LOG.warn("Failed to poll for push events", e); + } + } + + private void doBroadcastEvents() { + var clients = clientsRegistry.getClients(); + if (clients.isEmpty()) { + lastPullTimestamp = null; + lastSeenUuid = null; + return; + } + + if (lastPullTimestamp == null) { + lastPullTimestamp = system2.now(); + } + + var projectKeys = getClientsProjectKeys(clients); + + try (DbSession dbSession = dbClient.openSession(false)) { + var projectUuids = getProjectUuids(projectKeys, dbSession); + Deque<PushEventDto> events = getPushEvents(dbSession, projectUuids); + + LOG.debug("Received {} push events, attempting to broadcast to {} registered clients.", events.size(), + clients.size()); + + events.forEach(clientsRegistry::broadcastMessage); + + if (!events.isEmpty()) { + var last = events.getLast(); + lastPullTimestamp = last.getCreatedAt(); + lastSeenUuid = last.getUuid(); + } + } + } + + private static Set<String> getClientsProjectKeys(List<SonarLintClient> clients) { + return clients.stream() + .map(SonarLintClient::getClientProjectKeys) + .flatMap(Collection::stream) + .collect(Collectors.toSet()); + } + + private Deque<PushEventDto> getPushEvents(DbSession dbSession, Set<String> projectUuids) { + return dbClient.pushEventDao().selectChunkByProjectUuids(dbSession, projectUuids, lastPullTimestamp, lastSeenUuid, getPageSize()); + } + + @NotNull + private Set<String> getProjectUuids(Set<String> projectKeys, DbSession dbSession) { + return dbClient.projectDao().selectProjectsByKeys(dbSession, projectKeys) + .stream().map(ProjectDto::getUuid) + .collect(Collectors.toSet()); + } + + public long getInitialDelay() { + // two minutes default initial delay + return config.getLong(INITIAL_DELAY_IN_SECONDS).orElse(2 * 60L); + } + + public long getPeriod() { + // execute every 40 seconds + return config.getLong(PERIOD_IN_SECONDS).orElse(40L); + } + + public long getPageSize() { + // 20 events per 40 seconds + return config.getLong(PAGE_SIZE).orElse(20L); + } + + @Override + public void stop() { + // nothing to do + } + +} diff --git a/server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/sonarlint/SonarLintClientsRegistry.java b/server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/sonarlint/SonarLintClientsRegistry.java index d2508ada331..1d815f2371c 100644 --- a/server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/sonarlint/SonarLintClientsRegistry.java +++ b/server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/sonarlint/SonarLintClientsRegistry.java @@ -34,6 +34,7 @@ import org.sonar.core.util.issue.IssueChangeListener; import org.sonar.core.util.issue.IssueChangedEvent; import org.sonar.core.util.rule.RuleActivationListener; import org.sonar.core.util.rule.RuleSetChangedEvent; +import org.sonar.db.pushevent.PushEventDto; import org.sonar.server.exceptions.ForbiddenException; import org.sonar.server.pushapi.issues.IssueChangeBroadcastUtils; import org.sonar.server.pushapi.issues.IssueChangeEventsDistributor; @@ -86,6 +87,10 @@ public class SonarLintClientsRegistry implements RuleActivationListener, IssueCh LOG.debug("Removing SonarLint client"); } + public List<SonarLintClient> getClients() { + return clients; + } + public long countConnectedClients() { return clients.size(); } @@ -100,6 +105,11 @@ public class SonarLintClientsRegistry implements RuleActivationListener, IssueCh broadcastMessage(issueChangedEvent, IssueChangeBroadcastUtils.getFilterForEvent(issueChangedEvent)); } + public void broadcastMessage(PushEventDto event) { + // TODO:: different task for broadcasting event + LOG.info("received event: ({}, {}) ", event.getUuid(), event.getName()); + } + public void broadcastMessage(RuleSetChangedEvent event, Predicate<SonarLintClient> filter) { clients.stream().filter(filter).forEach(c -> { Set<String> projectKeysInterestingForClient = new HashSet<>(c.getClientProjectKeys()); diff --git a/server/sonar-webserver-pushapi/src/test/java/org/sonar/server/pushapi/ServerPushWsModuleTest.java b/server/sonar-webserver-pushapi/src/test/java/org/sonar/server/pushapi/ServerPushWsModuleTest.java index 9ef3dec04f2..87a78661495 100644 --- a/server/sonar-webserver-pushapi/src/test/java/org/sonar/server/pushapi/ServerPushWsModuleTest.java +++ b/server/sonar-webserver-pushapi/src/test/java/org/sonar/server/pushapi/ServerPushWsModuleTest.java @@ -28,7 +28,7 @@ public class ServerPushWsModuleTest { @Test public void verify_count_of_added_components() { ListContainer container = new ListContainer(); - new ServerPushWsModule().configure(container); + new ServerPushModule().configure(container); assertThat(container.getAddedObjects()).isNotEmpty(); } } diff --git a/server/sonar-webserver-pushapi/src/test/java/org/sonar/server/pushapi/scheduler/polling/PushEventPollExecutorServiceImplTest.java b/server/sonar-webserver-pushapi/src/test/java/org/sonar/server/pushapi/scheduler/polling/PushEventPollExecutorServiceImplTest.java new file mode 100644 index 00000000000..8bedaa7685f --- /dev/null +++ b/server/sonar-webserver-pushapi/src/test/java/org/sonar/server/pushapi/scheduler/polling/PushEventPollExecutorServiceImplTest.java @@ -0,0 +1,39 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.pushapi.scheduler.polling; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class PushEventPollExecutorServiceImplTest { + + @Test + public void create_executor() { + PushEventPollExecutorServiceImpl underTest = new PushEventPollExecutorServiceImpl(); + + assertThat(underTest.createThread(() -> { + })) + .extracting(Thread::getPriority, Thread::isDaemon, Thread::getName) + .containsExactly(Thread.MIN_PRIORITY, true, "PushEventPoll-%d"); + + } + +} diff --git a/server/sonar-webserver-pushapi/src/test/java/org/sonar/server/pushapi/scheduler/polling/PushEventPollSchedulerTest.java b/server/sonar-webserver-pushapi/src/test/java/org/sonar/server/pushapi/scheduler/polling/PushEventPollSchedulerTest.java new file mode 100644 index 00000000000..4d436f52971 --- /dev/null +++ b/server/sonar-webserver-pushapi/src/test/java/org/sonar/server/pushapi/scheduler/polling/PushEventPollSchedulerTest.java @@ -0,0 +1,209 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.pushapi.scheduler.polling; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.api.config.Configuration; +import org.sonar.api.impl.utils.TestSystem2; +import org.sonar.core.util.UuidFactoryFast; +import org.sonar.db.DbTester; +import org.sonar.db.pushevent.PushEventDto; +import org.sonar.server.pushapi.sonarlint.SonarLintClient; +import org.sonar.server.pushapi.sonarlint.SonarLintClientsRegistry; +import org.sonar.server.util.AbstractStoppableExecutorService; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Collections.emptyList; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class PushEventPollSchedulerTest { + + private final SonarLintClientsRegistry clientsRegistry = mock(SonarLintClientsRegistry.class); + + private static final long NOW = 1L; + private final TestSystem2 system2 = new TestSystem2().setNow(NOW); + private final Configuration config = mock(Configuration.class); + + @Rule + public DbTester db = DbTester.create(system2); + + private final SyncPushEventExecutorService executorService = new SyncPushEventExecutorService(); + + @Test + public void scheduler_should_be_resilient_to_failures() { + when(clientsRegistry.getClients()).thenThrow(new RuntimeException("I have a bad feelings about this")); + + var underTest = new PushEventPollScheduler(executorService, clientsRegistry, db.getDbClient(), system2, config); + underTest.start(); + + assertThatCode(executorService::runCommand) + .doesNotThrowAnyException(); + + verify(clientsRegistry, times(0)).broadcastMessage(any(PushEventDto.class)); + } + + @Test + public void nothing_to_broadcast_when_client_list_is_empty() { + when(clientsRegistry.getClients()).thenReturn(emptyList()); + + var underTest = new PushEventPollScheduler(executorService, clientsRegistry, db.getDbClient(), system2, config); + underTest.start(); + + executorService.runCommand(); + + verify(clientsRegistry, times(0)).broadcastMessage(any(PushEventDto.class)); + } + + @Test + public void nothing_to_broadcast_when_no_push_events() { + var project = db.components().insertPrivateProject(); + + var sonarLintClient = mock(SonarLintClient.class); + when(sonarLintClient.getClientProjectKeys()).thenReturn(Set.of(project.getDbKey())); + when(clientsRegistry.getClients()).thenReturn(List.of(sonarLintClient)); + + var underTest = new PushEventPollScheduler(executorService, clientsRegistry, db.getDbClient(), system2, config); + underTest.start(); + + executorService.runCommand(); + + verify(clientsRegistry, times(0)).broadcastMessage(any(PushEventDto.class)); + } + + @Test + public void broadcast_push_events() { + var project = db.components().insertPrivateProject(); + + system2.setNow(1L); + var sonarLintClient = mock(SonarLintClient.class); + when(sonarLintClient.getClientProjectKeys()).thenReturn(Set.of(project.getDbKey())); + when(clientsRegistry.getClients()).thenReturn(List.of(sonarLintClient)); + + var underTest = new PushEventPollScheduler(executorService, clientsRegistry, db.getDbClient(), system2, config); + underTest.start(); + executorService.runCommand(); + + verify(clientsRegistry, times(0)).broadcastMessage(any(PushEventDto.class)); + + system2.tick(); // tick=2 + generatePushEvent(project.uuid()); + generatePushEvent(project.uuid()); + + system2.tick(); // tick=3 + generatePushEvent(project.uuid()); + + underTest.start(); + executorService.runCommand(); + + verify(clientsRegistry, times(3)).broadcastMessage(any(PushEventDto.class)); + + system2.tick(); // tick=4 + generatePushEvent(project.uuid()); + generatePushEvent(project.uuid()); + + underTest.start(); + executorService.runCommand(); + verify(clientsRegistry, times(5)).broadcastMessage(any(PushEventDto.class)); + } + + @Test + public void broadcast_should_stop_polling_for_events_when_all_clients_unregister() { + var project = db.components().insertPrivateProject(); + + system2.setNow(1L); + var sonarLintClient = mock(SonarLintClient.class); + when(sonarLintClient.getClientProjectKeys()).thenReturn(Set.of(project.getDbKey())); + when(clientsRegistry.getClients()).thenReturn(List.of(sonarLintClient), emptyList()); + + var underTest = new PushEventPollScheduler(executorService, clientsRegistry, db.getDbClient(), system2, config); + underTest.start(); + executorService.runCommand(); + + verify(clientsRegistry, times(0)).broadcastMessage(any(PushEventDto.class)); + + system2.tick(); // tick=2 + generatePushEvent(project.uuid()); + + underTest.start(); + executorService.runCommand(); + + // all clients have been unregistered, nothing to broadcast + verify(clientsRegistry, times(0)).broadcastMessage(any(PushEventDto.class)); + } + + private PushEventDto generatePushEvent(String projectUuid) { + var event = db.getDbClient().pushEventDao().insert(db.getSession(), new PushEventDto() + .setName("Event") + .setUuid(UuidFactoryFast.getInstance().create()) + .setProjectUuid(projectUuid) + .setPayload("some-event".getBytes(UTF_8))); + db.commit(); + return event; + } + + private static class SyncPushEventExecutorService extends AbstractStoppableExecutorService<ScheduledExecutorService> + implements PushEventExecutorService { + + private Runnable command; + + public SyncPushEventExecutorService() { + super(null); + } + + public void runCommand() { + command.run(); + } + + @Override + public ScheduledFuture<?> scheduleAtFixedRate(Runnable command, long initialDelay, long period, TimeUnit unit) { + this.command = command; + return null; + } + + @Override + public ScheduledFuture<?> schedule(Runnable command, long delay, TimeUnit unit) { + return null; + } + + @Override + public <V> ScheduledFuture<V> schedule(Callable<V> callable, long delay, TimeUnit unit) { + return null; + } + + @Override + public ScheduledFuture<?> scheduleWithFixedDelay(Runnable command, long initialDelay, long delay, TimeUnit unit) { + return null; + } + + } + +} diff --git a/server/sonar-webserver-pushapi/src/test/java/org/sonar/server/pushapi/sonarlint/SonarLintClientsRegistryTest.java b/server/sonar-webserver-pushapi/src/test/java/org/sonar/server/pushapi/sonarlint/SonarLintClientsRegistryTest.java index 48cbb218411..084117b8ec1 100644 --- a/server/sonar-webserver-pushapi/src/test/java/org/sonar/server/pushapi/sonarlint/SonarLintClientsRegistryTest.java +++ b/server/sonar-webserver-pushapi/src/test/java/org/sonar/server/pushapi/sonarlint/SonarLintClientsRegistryTest.java @@ -78,10 +78,12 @@ public class SonarLintClientsRegistryTest { underTest.registerClient(sonarLintClient); assertThat(underTest.countConnectedClients()).isEqualTo(1); + assertThat(underTest.getClients()).contains(sonarLintClient); underTest.unregisterClient(sonarLintClient); assertThat(underTest.countConnectedClients()).isZero(); + assertThat(underTest.getClients()).isEmpty(); verify(sonarLintClient).close(); } diff --git a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java index 98562996585..d1f9dbfeeb0 100644 --- a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java +++ b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java @@ -184,7 +184,7 @@ import org.sonar.server.projectanalysis.ws.ProjectAnalysisWsModule; import org.sonar.server.projectlink.ws.ProjectLinksModule; import org.sonar.server.projecttag.ws.ProjectTagsWsModule; import org.sonar.server.property.InternalPropertiesImpl; -import org.sonar.server.pushapi.ServerPushWsModule; +import org.sonar.server.pushapi.ServerPushModule; import org.sonar.server.pushapi.issues.DistributedIssueChangeEventsDistributor; import org.sonar.server.pushapi.issues.IssueChangeEventServiceImpl; import org.sonar.server.pushapi.issues.StandaloneIssueChangeEventsDistributor; @@ -576,7 +576,7 @@ public class PlatformLevel4 extends PlatformLevel { MultipleAlmFeatureProvider.class, // ServerPush endpoints - new ServerPushWsModule(), + new ServerPushModule(), // Compute engine (must be after Views and Developer Cockpit) new ReportAnalysisFailureNotificationModule(), |