Sfoglia il codice sorgente

SONAR-16374 Add scheduled task to read PushEvents from DB

tags/9.6.0.59041
Jacek 1 anno fa
parent
commit
f20a4314ac
19 ha cambiato i file con 620 aggiunte e 7 eliminazioni
  1. 2
    3
      server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/pushevent/TaintVulnerabilityVisitor.java
  2. 1
    0
      server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistPushEventsStep.java
  3. 5
    0
      server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventDao.java
  4. 10
    0
      server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventDto.java
  5. 4
    0
      server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventMapper.java
  6. 44
    0
      server/sonar-db-dao/src/main/resources/org/sonar/db/pushevent/PushEventMapper.xml
  7. 1
    0
      server/sonar-db-dao/src/schema/schema-sq.ddl
  8. 62
    0
      server/sonar-db-dao/src/test/java/org/sonar/db/pushevent/PushEventDaoTest.java
  9. 1
    0
      server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v96/CreatePushEventsTable.java
  10. 5
    1
      server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/ServerPushModule.java
  11. 28
    0
      server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/scheduler/polling/PushEventExecutorService.java
  12. 44
    0
      server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/scheduler/polling/PushEventPollExecutorServiceImpl.java
  13. 150
    0
      server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/scheduler/polling/PushEventPollScheduler.java
  14. 10
    0
      server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/sonarlint/SonarLintClientsRegistry.java
  15. 1
    1
      server/sonar-webserver-pushapi/src/test/java/org/sonar/server/pushapi/ServerPushWsModuleTest.java
  16. 39
    0
      server/sonar-webserver-pushapi/src/test/java/org/sonar/server/pushapi/scheduler/polling/PushEventPollExecutorServiceImplTest.java
  17. 209
    0
      server/sonar-webserver-pushapi/src/test/java/org/sonar/server/pushapi/scheduler/polling/PushEventPollSchedulerTest.java
  18. 2
    0
      server/sonar-webserver-pushapi/src/test/java/org/sonar/server/pushapi/sonarlint/SonarLintClientsRegistryTest.java
  19. 2
    2
      server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java

+ 2
- 3
server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/pushevent/TaintVulnerabilityVisitor.java Vedi File

@@ -21,7 +21,6 @@ package org.sonar.ce.task.projectanalysis.pushevent;

import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import org.jetbrains.annotations.NotNull;
import org.sonar.api.rules.RuleType;
import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolder;
@@ -31,12 +30,12 @@ import org.sonar.ce.task.projectanalysis.issue.IssueVisitor;
import org.sonar.core.issue.DefaultIssue;
import org.sonar.db.protobuf.DbCommons;
import org.sonar.db.protobuf.DbIssues;
import org.sonar.server.issue.TaintChecker;

import static java.util.Objects.requireNonNull;
import static java.util.Objects.requireNonNullElse;

public class TaintVulnerabilityVisitor extends IssueVisitor {
private static final Set<String> TAINT_REPOSITORIES = Set.of("roslyn.sonaranalyzer.security.cs", "javasecurity", "jssecurity", "tssecurity", "phpsecurity", "pythonsecurity");

private final PushEventRepository pushEventRepository;
private final AnalysisMetadataHolder analysisMetadataHolder;
@@ -118,7 +117,7 @@ public class TaintVulnerabilityVisitor extends IssueVisitor {
}

private static boolean isTaintVulnerability(DefaultIssue issue) {
return TAINT_REPOSITORIES.contains(issue.getRuleKey().repository())
return TaintChecker.getTaintRepositories().contains(issue.getRuleKey().repository())
&& issue.getLocations() != null
&& !RuleType.SECURITY_HOTSPOT.equals(issue.type());
}

+ 1
- 0
server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistPushEventsStep.java Vedi File

@@ -73,6 +73,7 @@ public class PersistPushEventsStep implements ComputationStep {

private void pushEvent(DbSession dbSession, PushEvent<?> event) {
PushEventDto eventDto = new PushEventDto()
.setName(event.getName())
.setProjectUuid(treeRootHolder.getRoot().getUuid())
.setPayload(serializeIssueToPushEvent(event));
dbClient.pushEventDao().insert(dbSession, eventDto);

+ 5
- 0
server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventDao.java Vedi File

@@ -19,6 +19,7 @@
*/
package org.sonar.db.pushevent;

import java.util.Deque;
import java.util.Set;
import org.sonar.api.utils.System2;
import org.sonar.core.util.UuidFactory;
@@ -67,4 +68,8 @@ public class PushEventDao implements Dao {
return session.getMapper(PushEventMapper.class);
}

public Deque<PushEventDto> selectChunkByProjectUuids(DbSession dbSession, Set<String> projectUuids,
Long lastPullTimestamp, String lastSeenUuid, long count) {
return mapper(dbSession).selectChunkByProjectUuids(projectUuids, lastPullTimestamp, lastSeenUuid, count);
}
}

+ 10
- 0
server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventDto.java Vedi File

@@ -23,6 +23,7 @@ import javax.annotation.CheckForNull;

public class PushEventDto {
private String uuid;
private String name;
private String projectUuid;
private byte[] payload;
private Long createdAt;
@@ -40,6 +41,15 @@ public class PushEventDto {
return this;
}

public String getName() {
return name;
}

public PushEventDto setName(String name) {
this.name = name;
return this;
}

public String getProjectUuid() {
return projectUuid;
}

+ 4
- 0
server/sonar-db-dao/src/main/java/org/sonar/db/pushevent/PushEventMapper.java Vedi File

@@ -19,6 +19,7 @@
*/
package org.sonar.db.pushevent;

import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import javax.annotation.CheckForNull;
@@ -35,4 +36,7 @@ public interface PushEventMapper {

void deleteByUuids(@Param("pushEventUuids") List<String> pushEventUuids);

LinkedList<PushEventDto> selectChunkByProjectUuids(@Param("projectUuids") Set<String> projectUuids,
@Param("lastPullTimestamp") Long lastPullTimestamp,
@Param("lastSeenUuid") String lastSeenUuid, @Param("count") long count);
}

+ 44
- 0
server/sonar-db-dao/src/main/resources/org/sonar/db/pushevent/PushEventMapper.xml Vedi File

@@ -4,6 +4,7 @@

<sql id="pushEventColumns">
pe.uuid as uuid,
pe.name as name,
pe.project_uuid as projectUuid,
pe.payload as payload,
pe.created_at as createdAt
@@ -12,12 +13,14 @@
<insert id="insert" parameterType="map" useGeneratedKeys="false">
INSERT INTO push_events (
uuid,
name,
project_uuid,
payload,
created_at
)
VALUES (
#{uuid,jdbcType=VARCHAR},
#{name,jdbcType=VARCHAR},
#{projectUuid,jdbcType=VARCHAR},
#{payload,jdbcType=BLOB},
#{createdAt,jdbcType=BIGINT}
@@ -32,6 +35,47 @@
pe.uuid=#{uuid,jdbcType=VARCHAR}
</select>

<select id="selectChunkByProjectUuids" parameterType="map" resultType="PushEvent">
SELECT
<include refid="pushEventColumns"/>
FROM push_events pe
WHERE created_at &gt;= #{lastPullTimestamp,jdbcType=BIGINT}
AND ( created_at &gt; #{lastPullTimestamp,jdbcType=BIGINT} OR uuid &gt; #{lastSeenUuid,jdbcType=VARCHAR} )
AND pe.project_uuid in
<foreach collection="projectUuids" open="(" close=")" item="uuid" separator=",">
#{uuid,jdbcType=VARCHAR}
</foreach>
ORDER BY created_at, uuid
LIMIT #{count}
</select>

<select id="selectChunkByProjectUuids" parameterType="map" resultType="PushEvent" databaseId="mssql">
SELECT top (#{count,jdbcType=BIGINT})
<include refid="pushEventColumns"/>
FROM push_events pe
WHERE created_at &gt;= #{lastPullTimestamp,jdbcType=BIGINT}
AND ( created_at &gt; #{lastPullTimestamp,jdbcType=BIGINT} OR uuid &gt; #{lastSeenUuid,jdbcType=VARCHAR} )
AND pe.project_uuid in
<foreach collection="projectUuids" open="(" close=")" item="uuid" separator=",">
#{uuid,jdbcType=VARCHAR}
</foreach>
ORDER BY created_at, uuid
</select>

<select id="selectChunkByProjectUuids" parameterType="map" resultType="PushEvent" databaseId="oracle">
SELECT * FROM (SELECT
<include refid="pushEventColumns"/>
FROM push_events pe
WHERE created_at &gt;= #{lastPullTimestamp,jdbcType=BIGINT}
AND ( created_at &gt; #{lastPullTimestamp,jdbcType=BIGINT} OR uuid &gt; #{lastSeenUuid,jdbcType=VARCHAR} )
AND pe.project_uuid in
<foreach collection="projectUuids" open="(" close=")" item="uuid" separator=",">
#{uuid,jdbcType=VARCHAR}
</foreach>
ORDER BY created_at, uuid)
WHERE rownum &lt;= #{count,jdbcType=BIGINT}
</select>

<select id="selectUuidsOfExpiredEvents" parameterType="long" resultType="string">
SELECT
pe.uuid

+ 1
- 0
server/sonar-db-dao/src/schema/schema-sq.ddl Vedi File

@@ -742,6 +742,7 @@ CREATE INDEX "PROPERTIES_KEY" ON "PROPERTIES"("PROP_KEY" NULLS FIRST);

CREATE TABLE "PUSH_EVENTS"(
"UUID" CHARACTER VARYING(40) NOT NULL,
"NAME" CHARACTER VARYING(40) NOT NULL,
"PROJECT_UUID" CHARACTER VARYING(40) NOT NULL,
"PAYLOAD" BINARY LARGE OBJECT NOT NULL,
"CREATED_AT" BIGINT NOT NULL

+ 62
- 0
server/sonar-db-dao/src/test/java/org/sonar/db/pushevent/PushEventDaoTest.java Vedi File

@@ -20,9 +20,11 @@
package org.sonar.db.pushevent;

import java.util.Set;
import java.util.stream.IntStream;
import org.junit.Rule;
import org.junit.Test;
import org.sonar.api.impl.utils.TestSystem2;
import org.sonar.core.util.UuidFactoryFast;
import org.sonar.db.DbSession;
import org.sonar.db.DbTester;

@@ -45,11 +47,14 @@ public class PushEventDaoTest {

PushEventDto eventDtoFirst = new PushEventDto()
.setUuid("test-uuid")
.setName("Event")
.setProjectUuid("project-uuid")
.setPayload("some-event".getBytes(UTF_8));

PushEventDto eventDtoSecond = new PushEventDto()
.setProjectUuid("project-uuid")
.setName("Event")

.setPayload("some-event".getBytes(UTF_8));

underTest.insert(session, eventDtoFirst);
@@ -71,16 +76,19 @@ public class PushEventDaoTest {
@Test
public void select_expired_events() {
PushEventDto eventDtoFirst = new PushEventDto()
.setName("Event")
.setProjectUuid("project-uuid")
.setCreatedAt(1000L)
.setPayload("some-event".getBytes(UTF_8));

PushEventDto eventDtoSecond = new PushEventDto()
.setName("Event")
.setProjectUuid("project-uuid")
.setCreatedAt(1000L)
.setPayload("some-event".getBytes(UTF_8));

PushEventDto eventDtoThird = new PushEventDto()
.setName("Event")
.setProjectUuid("project-uuid")
.setCreatedAt(2000L)
.setPayload("some-event".getBytes(UTF_8));
@@ -97,11 +105,13 @@ public class PushEventDaoTest {
@Test
public void delete_events_in_batches() {
PushEventDto eventDtoFirst = new PushEventDto()
.setName("Event")
.setProjectUuid("project-uuid")
.setCreatedAt(1000L)
.setPayload("some-event".getBytes(UTF_8));

PushEventDto eventDtoSecond = new PushEventDto()
.setName("Event")
.setProjectUuid("project-uuid")
.setCreatedAt(1000L)
.setPayload("some-event".getBytes(UTF_8));
@@ -114,4 +124,56 @@ public class PushEventDaoTest {
assertThat(underTest.selectUuidsOfExpiredEvents(db.getSession(), 2000L)).isEmpty();
}

@Test
public void selectChunkByProjectKeys() {
system2.setNow(1L);
generatePushEvent("proj1");
system2.tick(); // tick=2
generatePushEvent("proj2");

system2.tick(); // tick=3
var eventDto4 = generatePushEvent("proj2");

var events = underTest.selectChunkByProjectUuids(session, Set.of("proj1", "proj2"), 2L, null, 10);

// tick=1 and tick=2 skipped
assertThat(events).extracting(PushEventDto::getUuid).containsExactly(eventDto4.getUuid());

system2.tick(); // tick=4
var eventDto5 = generatePushEvent("proj2");
var eventDto6 = generatePushEvent("proj2");

system2.tick(); // tick =5
var eventDto7 = generatePushEvent("proj2");

events = underTest.selectChunkByProjectUuids(session, Set.of("proj1", "proj2"), eventDto4.getCreatedAt(), eventDto4.getUuid(), 10);
assertThat(events).extracting(PushEventDto::getUuid).containsExactly(eventDto5.getUuid(), eventDto6.getUuid(), eventDto7.getUuid());
}

@Test
public void selectChunkByProjectKeys_pagination() {
system2.setNow(3L);

IntStream.range(1, 10)
.forEach(value -> generatePushEvent("event-" + value, "proj1"));

var events = underTest.selectChunkByProjectUuids(session, Set.of("proj1"), 1L, null, 3);
assertThat(events).extracting(PushEventDto::getUuid).containsExactly("event-1", "event-2", "event-3");

events = underTest.selectChunkByProjectUuids(session, Set.of("proj1"), 3L, "event-3", 3);
assertThat(events).extracting(PushEventDto::getUuid).containsExactly("event-4", "event-5", "event-6");
}

private PushEventDto generatePushEvent(String projectUuid) {
return generatePushEvent(UuidFactoryFast.getInstance().create(), projectUuid);
}

private PushEventDto generatePushEvent(String uuid, String projectUuid) {
return underTest.insert(session, new PushEventDto()
.setName("Event")
.setUuid(uuid)
.setProjectUuid(projectUuid)
.setPayload("some-event".getBytes(UTF_8)));
}

}

+ 1
- 0
server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v96/CreatePushEventsTable.java Vedi File

@@ -41,6 +41,7 @@ public class CreatePushEventsTable extends CreateTableChange {
public void execute(Context context, String tableName) throws SQLException {
context.execute(new CreateTableBuilder(getDialect(), tableName)
.addPkColumn(newVarcharColumnDefBuilder().setColumnName("uuid").setIsNullable(false).setLimit(UUID_SIZE).build())
.addColumn(newVarcharColumnDefBuilder().setColumnName("name").setIsNullable(false).setLimit(40).build())
.addColumn(newVarcharColumnDefBuilder().setColumnName("project_uuid").setIsNullable(false).setLimit(UUID_SIZE).build())
.addColumn(newBlobColumnDefBuilder().setColumnName("payload").setIsNullable(false).build())
.addColumn(newBigIntegerColumnDefBuilder().setColumnName("created_at").setIsNullable(false).build())

server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/ServerPushWsModule.java → server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/ServerPushModule.java Vedi File

@@ -20,6 +20,8 @@
package org.sonar.server.pushapi;

import org.sonar.core.platform.Module;
import org.sonar.server.pushapi.scheduler.polling.PushEventPollExecutorServiceImpl;
import org.sonar.server.pushapi.scheduler.polling.PushEventPollScheduler;
import org.sonar.server.pushapi.scheduler.purge.PushEventsPurgeExecutorServiceImpl;
import org.sonar.server.pushapi.scheduler.purge.PushEventsPurgeInitializer;
import org.sonar.server.pushapi.scheduler.purge.PushEventsPurgeSchedulerImpl;
@@ -27,7 +29,7 @@ import org.sonar.server.pushapi.sonarlint.SonarLintClientPermissionsValidator;
import org.sonar.server.pushapi.sonarlint.SonarLintClientsRegistry;
import org.sonar.server.pushapi.sonarlint.SonarLintPushAction;

public class ServerPushWsModule extends Module {
public class ServerPushModule extends Module {

@Override
protected void configureModule() {
@@ -37,6 +39,8 @@ public class ServerPushWsModule extends Module {
SonarLintClientsRegistry.class,
SonarLintPushAction.class,

PushEventPollExecutorServiceImpl.class,
PushEventPollScheduler.class,
// Push Events Purge
PushEventsPurgeSchedulerImpl.class,
PushEventsPurgeExecutorServiceImpl.class,

+ 28
- 0
server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/scheduler/polling/PushEventExecutorService.java Vedi File

@@ -0,0 +1,28 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.pushapi.scheduler.polling;

import java.util.concurrent.ScheduledExecutorService;
import org.sonar.api.server.ServerSide;

@ServerSide
public interface PushEventExecutorService extends ScheduledExecutorService {

}

+ 44
- 0
server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/scheduler/polling/PushEventPollExecutorServiceImpl.java Vedi File

@@ -0,0 +1,44 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.pushapi.scheduler.polling;

import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import org.sonar.api.server.ServerSide;
import org.sonar.server.util.AbstractStoppableScheduledExecutorServiceImpl;

import static java.lang.Thread.MIN_PRIORITY;
import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;

public class PushEventPollExecutorServiceImpl extends
AbstractStoppableScheduledExecutorServiceImpl<ScheduledExecutorService> implements PushEventExecutorService {

public PushEventPollExecutorServiceImpl() {
super(newSingleThreadScheduledExecutor(PushEventPollExecutorServiceImpl::createThread));
}

static Thread createThread(Runnable r) {
Thread thread = Executors.defaultThreadFactory().newThread(r);
thread.setName("PushEventPoll-%d");
thread.setPriority(MIN_PRIORITY);
thread.setDaemon(true);
return thread;
}
}

+ 150
- 0
server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/scheduler/polling/PushEventPollScheduler.java Vedi File

@@ -0,0 +1,150 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.pushapi.scheduler.polling;

import java.util.Collection;
import java.util.Deque;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.jetbrains.annotations.NotNull;
import org.sonar.api.Startable;
import org.sonar.api.config.Configuration;
import org.sonar.api.server.ServerSide;
import org.sonar.api.utils.System2;
import org.sonar.api.utils.log.Logger;
import org.sonar.api.utils.log.Loggers;
import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
import org.sonar.db.project.ProjectDto;
import org.sonar.db.pushevent.PushEventDto;
import org.sonar.server.pushapi.sonarlint.SonarLintClient;
import org.sonar.server.pushapi.sonarlint.SonarLintClientsRegistry;

@ServerSide
public class PushEventPollScheduler implements Startable {

private static final Logger LOG = Loggers.get(PushEventPollScheduler.class);

private static final String INITIAL_DELAY_IN_SECONDS = "sonar.pushevents.polling.initial.delay";
private static final String PERIOD_IN_SECONDS = "sonar.pushevents.polling.period";
private static final String PAGE_SIZE = "sonar.pushevents.polling.page.size";

private final PushEventExecutorService executorService;
private final SonarLintClientsRegistry clientsRegistry;
private final DbClient dbClient;
private final System2 system2;
private final Configuration config;
private Long lastPullTimestamp = null;
private String lastSeenUuid = null;

public PushEventPollScheduler(PushEventExecutorService executorService, SonarLintClientsRegistry clientsRegistry,
DbClient dbClient, System2 system2, Configuration config) {
this.executorService = executorService;
this.clientsRegistry = clientsRegistry;
this.dbClient = dbClient;
this.system2 = system2;
this.config = config;
}

@Override
public void start() {
this.executorService.scheduleAtFixedRate(this::tryBroadcastEvents, getInitialDelay(), getPeriod(), TimeUnit.SECONDS);
}

private void tryBroadcastEvents() {
try {
doBroadcastEvents();
} catch (Exception e) {
LOG.warn("Failed to poll for push events", e);
}
}

private void doBroadcastEvents() {
var clients = clientsRegistry.getClients();
if (clients.isEmpty()) {
lastPullTimestamp = null;
lastSeenUuid = null;
return;
}

if (lastPullTimestamp == null) {
lastPullTimestamp = system2.now();
}

var projectKeys = getClientsProjectKeys(clients);

try (DbSession dbSession = dbClient.openSession(false)) {
var projectUuids = getProjectUuids(projectKeys, dbSession);
Deque<PushEventDto> events = getPushEvents(dbSession, projectUuids);

LOG.debug("Received {} push events, attempting to broadcast to {} registered clients.", events.size(),
clients.size());

events.forEach(clientsRegistry::broadcastMessage);

if (!events.isEmpty()) {
var last = events.getLast();
lastPullTimestamp = last.getCreatedAt();
lastSeenUuid = last.getUuid();
}
}
}

private static Set<String> getClientsProjectKeys(List<SonarLintClient> clients) {
return clients.stream()
.map(SonarLintClient::getClientProjectKeys)
.flatMap(Collection::stream)
.collect(Collectors.toSet());
}

private Deque<PushEventDto> getPushEvents(DbSession dbSession, Set<String> projectUuids) {
return dbClient.pushEventDao().selectChunkByProjectUuids(dbSession, projectUuids, lastPullTimestamp, lastSeenUuid, getPageSize());
}

@NotNull
private Set<String> getProjectUuids(Set<String> projectKeys, DbSession dbSession) {
return dbClient.projectDao().selectProjectsByKeys(dbSession, projectKeys)
.stream().map(ProjectDto::getUuid)
.collect(Collectors.toSet());
}

public long getInitialDelay() {
// two minutes default initial delay
return config.getLong(INITIAL_DELAY_IN_SECONDS).orElse(2 * 60L);
}

public long getPeriod() {
// execute every 40 seconds
return config.getLong(PERIOD_IN_SECONDS).orElse(40L);
}

public long getPageSize() {
// 20 events per 40 seconds
return config.getLong(PAGE_SIZE).orElse(20L);
}

@Override
public void stop() {
// nothing to do
}

}

+ 10
- 0
server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/sonarlint/SonarLintClientsRegistry.java Vedi File

@@ -34,6 +34,7 @@ import org.sonar.core.util.issue.IssueChangeListener;
import org.sonar.core.util.issue.IssueChangedEvent;
import org.sonar.core.util.rule.RuleActivationListener;
import org.sonar.core.util.rule.RuleSetChangedEvent;
import org.sonar.db.pushevent.PushEventDto;
import org.sonar.server.exceptions.ForbiddenException;
import org.sonar.server.pushapi.issues.IssueChangeBroadcastUtils;
import org.sonar.server.pushapi.issues.IssueChangeEventsDistributor;
@@ -86,6 +87,10 @@ public class SonarLintClientsRegistry implements RuleActivationListener, IssueCh
LOG.debug("Removing SonarLint client");
}

public List<SonarLintClient> getClients() {
return clients;
}

public long countConnectedClients() {
return clients.size();
}
@@ -100,6 +105,11 @@ public class SonarLintClientsRegistry implements RuleActivationListener, IssueCh
broadcastMessage(issueChangedEvent, IssueChangeBroadcastUtils.getFilterForEvent(issueChangedEvent));
}

public void broadcastMessage(PushEventDto event) {
// TODO:: different task for broadcasting event
LOG.info("received event: ({}, {}) ", event.getUuid(), event.getName());
}

public void broadcastMessage(RuleSetChangedEvent event, Predicate<SonarLintClient> filter) {
clients.stream().filter(filter).forEach(c -> {
Set<String> projectKeysInterestingForClient = new HashSet<>(c.getClientProjectKeys());

+ 1
- 1
server/sonar-webserver-pushapi/src/test/java/org/sonar/server/pushapi/ServerPushWsModuleTest.java Vedi File

@@ -28,7 +28,7 @@ public class ServerPushWsModuleTest {
@Test
public void verify_count_of_added_components() {
ListContainer container = new ListContainer();
new ServerPushWsModule().configure(container);
new ServerPushModule().configure(container);
assertThat(container.getAddedObjects()).isNotEmpty();
}
}

+ 39
- 0
server/sonar-webserver-pushapi/src/test/java/org/sonar/server/pushapi/scheduler/polling/PushEventPollExecutorServiceImplTest.java Vedi File

@@ -0,0 +1,39 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.pushapi.scheduler.polling;

import org.junit.Test;

import static org.assertj.core.api.Assertions.assertThat;

public class PushEventPollExecutorServiceImplTest {

@Test
public void create_executor() {
PushEventPollExecutorServiceImpl underTest = new PushEventPollExecutorServiceImpl();

assertThat(underTest.createThread(() -> {
}))
.extracting(Thread::getPriority, Thread::isDaemon, Thread::getName)
.containsExactly(Thread.MIN_PRIORITY, true, "PushEventPoll-%d");

}

}

+ 209
- 0
server/sonar-webserver-pushapi/src/test/java/org/sonar/server/pushapi/scheduler/polling/PushEventPollSchedulerTest.java Vedi File

@@ -0,0 +1,209 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.pushapi.scheduler.polling;

import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import org.junit.Rule;
import org.junit.Test;
import org.sonar.api.config.Configuration;
import org.sonar.api.impl.utils.TestSystem2;
import org.sonar.core.util.UuidFactoryFast;
import org.sonar.db.DbTester;
import org.sonar.db.pushevent.PushEventDto;
import org.sonar.server.pushapi.sonarlint.SonarLintClient;
import org.sonar.server.pushapi.sonarlint.SonarLintClientsRegistry;
import org.sonar.server.util.AbstractStoppableExecutorService;

import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Collections.emptyList;
import static org.assertj.core.api.Assertions.assertThatCode;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

public class PushEventPollSchedulerTest {

private final SonarLintClientsRegistry clientsRegistry = mock(SonarLintClientsRegistry.class);

private static final long NOW = 1L;
private final TestSystem2 system2 = new TestSystem2().setNow(NOW);
private final Configuration config = mock(Configuration.class);

@Rule
public DbTester db = DbTester.create(system2);

private final SyncPushEventExecutorService executorService = new SyncPushEventExecutorService();

@Test
public void scheduler_should_be_resilient_to_failures() {
when(clientsRegistry.getClients()).thenThrow(new RuntimeException("I have a bad feelings about this"));

var underTest = new PushEventPollScheduler(executorService, clientsRegistry, db.getDbClient(), system2, config);
underTest.start();

assertThatCode(executorService::runCommand)
.doesNotThrowAnyException();

verify(clientsRegistry, times(0)).broadcastMessage(any(PushEventDto.class));
}

@Test
public void nothing_to_broadcast_when_client_list_is_empty() {
when(clientsRegistry.getClients()).thenReturn(emptyList());

var underTest = new PushEventPollScheduler(executorService, clientsRegistry, db.getDbClient(), system2, config);
underTest.start();

executorService.runCommand();

verify(clientsRegistry, times(0)).broadcastMessage(any(PushEventDto.class));
}

@Test
public void nothing_to_broadcast_when_no_push_events() {
var project = db.components().insertPrivateProject();

var sonarLintClient = mock(SonarLintClient.class);
when(sonarLintClient.getClientProjectKeys()).thenReturn(Set.of(project.getDbKey()));
when(clientsRegistry.getClients()).thenReturn(List.of(sonarLintClient));

var underTest = new PushEventPollScheduler(executorService, clientsRegistry, db.getDbClient(), system2, config);
underTest.start();

executorService.runCommand();

verify(clientsRegistry, times(0)).broadcastMessage(any(PushEventDto.class));
}

@Test
public void broadcast_push_events() {
var project = db.components().insertPrivateProject();

system2.setNow(1L);
var sonarLintClient = mock(SonarLintClient.class);
when(sonarLintClient.getClientProjectKeys()).thenReturn(Set.of(project.getDbKey()));
when(clientsRegistry.getClients()).thenReturn(List.of(sonarLintClient));

var underTest = new PushEventPollScheduler(executorService, clientsRegistry, db.getDbClient(), system2, config);
underTest.start();
executorService.runCommand();

verify(clientsRegistry, times(0)).broadcastMessage(any(PushEventDto.class));

system2.tick(); // tick=2
generatePushEvent(project.uuid());
generatePushEvent(project.uuid());

system2.tick(); // tick=3
generatePushEvent(project.uuid());

underTest.start();
executorService.runCommand();

verify(clientsRegistry, times(3)).broadcastMessage(any(PushEventDto.class));

system2.tick(); // tick=4
generatePushEvent(project.uuid());
generatePushEvent(project.uuid());

underTest.start();
executorService.runCommand();
verify(clientsRegistry, times(5)).broadcastMessage(any(PushEventDto.class));
}

@Test
public void broadcast_should_stop_polling_for_events_when_all_clients_unregister() {
var project = db.components().insertPrivateProject();

system2.setNow(1L);
var sonarLintClient = mock(SonarLintClient.class);
when(sonarLintClient.getClientProjectKeys()).thenReturn(Set.of(project.getDbKey()));
when(clientsRegistry.getClients()).thenReturn(List.of(sonarLintClient), emptyList());

var underTest = new PushEventPollScheduler(executorService, clientsRegistry, db.getDbClient(), system2, config);
underTest.start();
executorService.runCommand();

verify(clientsRegistry, times(0)).broadcastMessage(any(PushEventDto.class));

system2.tick(); // tick=2
generatePushEvent(project.uuid());

underTest.start();
executorService.runCommand();

// all clients have been unregistered, nothing to broadcast
verify(clientsRegistry, times(0)).broadcastMessage(any(PushEventDto.class));
}

private PushEventDto generatePushEvent(String projectUuid) {
var event = db.getDbClient().pushEventDao().insert(db.getSession(), new PushEventDto()
.setName("Event")
.setUuid(UuidFactoryFast.getInstance().create())
.setProjectUuid(projectUuid)
.setPayload("some-event".getBytes(UTF_8)));
db.commit();
return event;
}

private static class SyncPushEventExecutorService extends AbstractStoppableExecutorService<ScheduledExecutorService>
implements PushEventExecutorService {

private Runnable command;

public SyncPushEventExecutorService() {
super(null);
}

public void runCommand() {
command.run();
}

@Override
public ScheduledFuture<?> scheduleAtFixedRate(Runnable command, long initialDelay, long period, TimeUnit unit) {
this.command = command;
return null;
}

@Override
public ScheduledFuture<?> schedule(Runnable command, long delay, TimeUnit unit) {
return null;
}

@Override
public <V> ScheduledFuture<V> schedule(Callable<V> callable, long delay, TimeUnit unit) {
return null;
}

@Override
public ScheduledFuture<?> scheduleWithFixedDelay(Runnable command, long initialDelay, long delay, TimeUnit unit) {
return null;
}

}

}

+ 2
- 0
server/sonar-webserver-pushapi/src/test/java/org/sonar/server/pushapi/sonarlint/SonarLintClientsRegistryTest.java Vedi File

@@ -78,10 +78,12 @@ public class SonarLintClientsRegistryTest {
underTest.registerClient(sonarLintClient);

assertThat(underTest.countConnectedClients()).isEqualTo(1);
assertThat(underTest.getClients()).contains(sonarLintClient);

underTest.unregisterClient(sonarLintClient);

assertThat(underTest.countConnectedClients()).isZero();
assertThat(underTest.getClients()).isEmpty();
verify(sonarLintClient).close();
}


+ 2
- 2
server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java Vedi File

@@ -184,7 +184,7 @@ import org.sonar.server.projectanalysis.ws.ProjectAnalysisWsModule;
import org.sonar.server.projectlink.ws.ProjectLinksModule;
import org.sonar.server.projecttag.ws.ProjectTagsWsModule;
import org.sonar.server.property.InternalPropertiesImpl;
import org.sonar.server.pushapi.ServerPushWsModule;
import org.sonar.server.pushapi.ServerPushModule;
import org.sonar.server.pushapi.issues.DistributedIssueChangeEventsDistributor;
import org.sonar.server.pushapi.issues.IssueChangeEventServiceImpl;
import org.sonar.server.pushapi.issues.StandaloneIssueChangeEventsDistributor;
@@ -576,7 +576,7 @@ public class PlatformLevel4 extends PlatformLevel {
MultipleAlmFeatureProvider.class,

// ServerPush endpoints
new ServerPushWsModule(),
new ServerPushModule(),

// Compute engine (must be after Views and Developer Cockpit)
new ReportAnalysisFailureNotificationModule(),

Loading…
Annulla
Salva