*/
package org.sonar.ce.monitoring;
+import java.util.Optional;
import org.sonar.api.ce.ComputeEngineSide;
@ComputeEngineSide
long addSuccess(long processingTime);
/**
- * Adds 1 to the count of batch reports which processing ended with an error and removes 1 from the count of batch
- * reports under processing. Adds the specified time to the processing time counter.
+ * Adds 1 to the count of tasks which processing ended with an error and removes 1 from the count of tasks
+ * under processing. Adds the specified time to the processing time counter.
*
* @param processingTime duration of processing in ms
*
- * @return the new count of batch reports which processing ended with an error
+ * @return the new count of tasks which processing ended with an error
*
* @see #getErrorCount()
* @see #getInProgressCount()
long addError(long processingTime);
/**
- * Count of batch reports waiting for processing since startup, including reports received before instance startup.
+ * Number of pending tasks, including tasks received before instance startup.
*/
long getPendingCount();
/**
- * Count of batch reports under processing.
+ * The age, in ms, of the oldest pending task.
+ */
+ Optional<Long> getLongestTimePending();
+
+ /**
+ * Count of tasks under processing.
*/
long getInProgressCount();
/**
- * Count of batch reports which processing ended with an error since instance startup.
+ * Count of tasks which processing ended with an error since instance startup.
*/
long getErrorCount();
/**
- * Count of batch reports which processing ended successfully since instance startup.
+ * Count of tasks which processing ended successfully since instance startup.
*/
long getSuccessCount();
/**
- * Time spent processing batch reports since startup, in milliseconds.
+ * Time spent processing tasks since startup, in milliseconds.
*/
long getProcessingTime();
import java.util.Optional;
import java.util.concurrent.atomic.AtomicLong;
+import org.sonar.api.utils.System2;
import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
import org.sonar.db.ce.CeQueueDto;
public class CEQueueStatusImpl implements CEQueueStatus {
private final DbClient dbClient;
+ private final System2 system;
private final AtomicLong inProgress = new AtomicLong(0);
private final AtomicLong error = new AtomicLong(0);
private final AtomicLong success = new AtomicLong(0);
private final AtomicLong processingTime = new AtomicLong(0);
- public CEQueueStatusImpl(DbClient dbClient) {
+ public CEQueueStatusImpl(DbClient dbClient, System2 system) {
this.dbClient = dbClient;
+ this.system = system;
}
@Override
}
}
+ @Override
+ public Optional<Long> getLongestTimePending() {
+ try (DbSession dbSession = dbClient.openSession(false)) {
+ return dbClient.ceQueueDao().selectCreationDateOfOldestPendingByMainComponentUuid(dbSession, null)
+ .map(creationDate -> system.now() - creationDate);
+ }
+ }
+
@Override
public boolean areWorkersPaused() {
try (DbSession dbSession = dbClient.openSession(false)) {
package org.sonar.ce.monitoring;
import java.util.List;
+import java.util.Optional;
public interface CeTasksMBean {
String OBJECT_NAME = "SonarQube:name=ComputeEngineTasks";
/**
- * Count of batch reports waiting for processing since startup, including reports received before instance startup.
+ * Number of pending tasks, including tasks received before instance startup.
*/
long getPendingCount();
/**
- * Count of batch reports under processing.
+ * The age, in ms, of the oldest pending task.
+ */
+ Optional<Long> getLongestTimePending();
+
+ /**
+ * Count of tasks under processing.
*/
long getInProgressCount();
/**
- * Count of batch reports which processing ended with an error since instance startup.
+ * Count of tasks which processing ended with an error since instance startup.
*/
long getErrorCount();
/**
- * Count of batch reports which processing ended successfully since instance startup.
+ * Count of tasks which processing ended successfully since instance startup.
*/
long getSuccessCount();
/**
- * Time spent processing reports since startup, in milliseconds.
+ * Time spent processing tasks since startup, in milliseconds.
*/
long getProcessingTime();
package org.sonar.ce.monitoring;
import java.util.List;
+import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import org.picocontainer.Startable;
return queueStatus.getPendingCount();
}
+ @Override
+ public Optional<Long> getLongestTimePending() {
+ return queueStatus.getLongestTimePending();
+ }
+
@Override
public long getInProgressCount() {
return queueStatus.getInProgressCount();
ProtobufSystemInfo.Section.Builder builder = ProtobufSystemInfo.Section.newBuilder();
builder.setName("Compute Engine Tasks");
builder.addAttributesBuilder().setKey("Pending").setLongValue(getPendingCount()).build();
+ builder.addAttributesBuilder().setKey("Longest Time Pending (ms)").setLongValue(getLongestTimePending().orElse(0L)).build();
builder.addAttributesBuilder().setKey("In Progress").setLongValue(getInProgressCount()).build();
builder.addAttributesBuilder().setKey("Processed With Error").setLongValue(getErrorCount()).build();
builder.addAttributesBuilder().setKey("Processed With Success").setLongValue(getSuccessCount()).build();
*/
package org.sonar.ce.monitoring;
+import org.sonar.api.utils.System2;
import org.sonar.db.DbClient;
public class DistributedCEQueueStatusImpl extends CEQueueStatusImpl {
- public DistributedCEQueueStatusImpl(DbClient dbClient) {
- super(dbClient);
+ public DistributedCEQueueStatusImpl(DbClient dbClient, System2 system2) {
+ super(dbClient, system2);
}
@Override
import java.util.concurrent.TimeUnit;
import org.junit.After;
import org.junit.Test;
+import org.sonar.api.utils.System2;
import org.sonar.db.DbClient;
import static org.assertj.core.api.Assertions.assertThat;
return new Thread(r, CEQueueStatusImplConcurrentTest.class.getSimpleName() + cnt++);
}
});
- private CEQueueStatusImpl underTest = new CEQueueStatusImpl(mock(DbClient.class));
+ private CEQueueStatusImpl underTest = new CEQueueStatusImpl(mock(DbClient.class), mock(System2.class));
@After
public void tearDown() {
import org.junit.Test;
import org.mockito.Mockito;
+import org.sonar.api.utils.System2;
import org.sonar.db.DbClient;
import org.sonar.db.ce.CeQueueDto;
import static org.mockito.Mockito.when;
public class CEQueueStatusImplTest extends CommonCEQueueStatusImplTest {
- private CEQueueStatusImpl underTest = new CEQueueStatusImpl(getDbClient());
+ private CEQueueStatusImpl underTest = new CEQueueStatusImpl(getDbClient(), mock(System2.class));
public CEQueueStatusImplTest() {
super(mock(DbClient.class, Mockito.RETURNS_DEEP_STUBS));
import com.google.common.collect.ImmutableSet;
import java.lang.management.ManagementFactory;
import java.util.List;
+import java.util.Optional;
import java.util.Random;
import java.util.Set;
import java.util.stream.Collectors;
public class CeTasksMBeanImplTest {
private static final long PENDING_COUNT = 2;
+ private static final Optional<Long> PENDING_TIME = Optional.of(10_000L);
private static final long IN_PROGRESS_COUNT = 5;
private static final long ERROR_COUNT = 10;
private static final long SUCCESS_COUNT = 13;
@Test
public void get_methods_delegate_to_the_CEQueueStatus_instance() {
assertThat(underTest.getPendingCount()).isEqualTo(PENDING_COUNT);
+ assertThat(underTest.getLongestTimePending()).isEqualTo(PENDING_TIME);
assertThat(underTest.getInProgressCount()).isEqualTo(IN_PROGRESS_COUNT);
assertThat(underTest.getErrorCount()).isEqualTo(ERROR_COUNT);
assertThat(underTest.getSuccessCount()).isEqualTo(SUCCESS_COUNT);
public void export_system_info() {
ProtobufSystemInfo.Section section = underTest.toProtobuf();
assertThat(section.getName()).isEqualTo("Compute Engine Tasks");
- assertThat(section.getAttributesCount()).isEqualTo(8);
+ assertThat(section.getAttributesCount()).isEqualTo(9);
}
private static class DumbCEQueueStatus implements CEQueueStatus {
return PENDING_COUNT;
}
+ @Override
+ public Optional<Long> getLongestTimePending() {
+ return PENDING_TIME;
+ }
+
@Override
public long addInProgress() {
return methodNotImplemented();
}
}
+
private static class DumbCeConfiguration implements CeConfiguration {
@Override
import org.junit.Test;
import org.mockito.Mockito;
+import org.sonar.api.utils.System2;
import org.sonar.db.DbClient;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.verifyZeroInteractions;
public class DistributedCEQueueStatusImplTest extends CommonCEQueueStatusImplTest {
- private DistributedCEQueueStatusImpl underTest = new DistributedCEQueueStatusImpl(getDbClient());
+ private DistributedCEQueueStatusImpl underTest = new DistributedCEQueueStatusImpl(getDbClient(), mock(System2.class));
public DistributedCEQueueStatusImplTest() {
super(mock(DbClient.class, Mockito.RETURNS_DEEP_STUBS));
private DbSession session = db.getSession();
private UuidFactory uuidFactory = UuidFactoryImpl.INSTANCE;
- private CEQueueStatus queueStatus = new CEQueueStatusImpl(db.getDbClient());
+ private CEQueueStatus queueStatus = new CEQueueStatusImpl(db.getDbClient(), mock(System2.class));
private DefaultOrganizationProvider defaultOrganizationProvider = mock(DefaultOrganizationProvider.class);
private ComputeEngineStatus computeEngineStatus = mock(ComputeEngineStatus.class);
private InternalCeQueue underTest = new InternalCeQueueImpl(system2, db.getDbClient(), uuidFactory, queueStatus, defaultOrganizationProvider, computeEngineStatus);
return mapper(dbSession).countByStatusAndMainComponentUuid(status, mainComponentUuid);
}
+ public Optional<Long> selectCreationDateOfOldestPendingByMainComponentUuid(DbSession dbSession, @Nullable String mainComponentUuid) {
+ return Optional.ofNullable(mapper(dbSession).selectCreationDateOfOldestPendingByMainComponentUuid(mainComponentUuid));
+ }
+
/**
* Counts entries in the queue with the specified status for each specified main component uuid.
- *
* The returned map doesn't contain any entry for main component uuids for which there is no entry in the queue (ie.
* all entries have a value >= 0).
*/
int countByStatusAndMainComponentUuid(@Param("status") CeQueueDto.Status status, @Nullable @Param("mainComponentUuid") String mainComponentUuid);
+ @CheckForNull
+ Long selectCreationDateOfOldestPendingByMainComponentUuid(@Nullable @Param("mainComponentUuid") String mainComponentUuid);
+
List<QueueCount> countByStatusAndMainComponentUuids(@Param("status") CeQueueDto.Status status, @Param("mainComponentUuids") List<String> mainComponentUuids);
void insert(CeQueueDto dto);
</if>
</select>
+ <select id="selectCreationDateOfOldestPendingByMainComponentUuid" parameterType="map" resultType="Long">
+ select
+ min(created_at)
+ from
+ ce_queue
+ where
+ status='PENDING'
+ <if test="mainComponentUuid!=null">
+ and main_component_uuid=#{mainComponentUuid,jdbcType=VARCHAR}
+ </if>
+ </select>
+
<select id="countByStatusAndMainComponentUuids" resultType="org.sonar.db.ce.QueueCount">
select
main_component_uuid as mainComponentUuid,
import java.util.Map;
import java.util.Optional;
import java.util.Random;
+import java.util.function.Consumer;
import java.util.stream.Stream;
import javax.annotation.Nullable;
import org.junit.Rule;
.containsOnly("p1", "p2", "p3");
}
+ @Test
+ public void selectCreationDateOfOldestPendingByMainComponentUuid_on_any_component_returns_date() {
+ long time = alwaysIncreasingSystem2.now() + 10_000;
+ insertPending("p1", dto -> {
+ dto.setCreatedAt(time);
+ dto.setUpdatedAt(time+500);
+ dto.setMainComponentUuid("c1");
+ });
+ insertPending("p2", dto -> {
+ dto.setCreatedAt(time + 1000);
+ dto.setUpdatedAt(time + 2000);
+ dto.setMainComponentUuid("c2");
+ });
+
+ makeInProgress("w1", alwaysIncreasingSystem2.now(), insertPending("i1", dto -> dto.setMainComponentUuid("c3")));
+
+ assertThat(underTest.selectCreationDateOfOldestPendingByMainComponentUuid(db.getSession(), null))
+ .isEqualTo(Optional.of(time));
+ }
+
+ @Test
+ public void selectCreationDateOfOldestPendingByMainComponentUuid_on_specific_component_returns_date() {
+ long time = alwaysIncreasingSystem2.now() + 10_000;
+ insertPending("p1", dto -> {
+ dto.setCreatedAt(time);
+ dto.setUpdatedAt(time+500);
+ dto.setMainComponentUuid("c2");
+ });
+ insertPending("p2", dto -> {
+ dto.setCreatedAt(time + 2000);
+ dto.setUpdatedAt(time + 3000);
+ dto.setMainComponentUuid("c1");
+ });
+ insertPending("p3", dto -> {
+ dto.setCreatedAt(time + 4000);
+ dto.setUpdatedAt(time + 5000);
+ dto.setMainComponentUuid("c1");
+ });
+
+ makeInProgress("w1", alwaysIncreasingSystem2.now(), insertPending("i1", dto -> dto.setMainComponentUuid("c1")));
+
+ assertThat(underTest.selectCreationDateOfOldestPendingByMainComponentUuid(db.getSession(), "c1"))
+ .isEqualTo(Optional.of(time+2000));
+ }
+
+ @Test
+ public void selectCreationDateOfOldestPendingByMainComponentUuid_returns_empty_when_no_pending_tasks() {
+ makeInProgress("w1", alwaysIncreasingSystem2.now(), insertPending("i1"));
+ assertThat(underTest.selectCreationDateOfOldestPendingByMainComponentUuid(db.getSession(), null))
+ .isEmpty();
+ }
+
@Test
public void selectWornout_returns_task_pending_with_a_non_null_startedAt() {
insertPending("p1");
}
private CeQueueDto insertPending(String uuid) {
+ return insertPending(uuid, (Consumer<CeQueueDto>) null);
+ }
+
+ private CeQueueDto insertPending(String uuid, @Nullable Consumer<CeQueueDto> dtoConsumer) {
CeQueueDto dto = new CeQueueDto();
dto.setUuid(uuid);
dto.setTaskType(CeTaskTypes.REPORT);
dto.setStatus(PENDING);
dto.setSubmitterUuid("henri");
+ if (dtoConsumer != null) {
+ dtoConsumer.accept(dto);
+ }
underTestAlwaysIncreasingSystem2.insert(db.getSession(), dto);
db.getSession().commit();
return dto;
}
private void verifyCeQueueStatuses(String taskUuid1, CeQueueDto.Status taskStatus1, String taskUuid2, CeQueueDto.Status taskStatus2) {
- verifyCeQueueStatuses(new String[]{taskUuid1, taskUuid2}, new CeQueueDto.Status[]{taskStatus1, taskStatus2});
+ verifyCeQueueStatuses(new String[] {taskUuid1, taskUuid2}, new CeQueueDto.Status[] {taskStatus1, taskStatus2});
}
private void verifyCeQueueStatuses(String[] taskUuids, CeQueueDto.Status[] statuses) {
*/
package org.sonar.server.ce.ws;
-import com.google.common.base.Optional;
+import java.util.Optional;
+import javax.annotation.CheckForNull;
+import javax.annotation.Nullable;
import org.sonar.api.server.ws.Change;
import org.sonar.api.server.ws.Response;
import org.sonar.api.server.ws.WebService;
+import org.sonar.api.utils.System2;
import org.sonar.api.web.UserRole;
import org.sonar.core.util.Uuids;
import org.sonar.db.DbClient;
private final UserSession userSession;
private final DbClient dbClient;
private final ComponentFinder componentFinder;
+ private final System2 system2;
- public ActivityStatusAction(UserSession userSession, DbClient dbClient, ComponentFinder componentFinder) {
+ public ActivityStatusAction(UserSession userSession, DbClient dbClient, ComponentFinder componentFinder, System2 system2) {
this.userSession = userSession;
this.dbClient = dbClient;
this.componentFinder = componentFinder;
+ this.system2 = system2;
}
@Override
public void define(WebService.NewController controller) {
WebService.NewAction action = controller
.createAction("activity_status")
- .setDescription("Return CE activity related metrics.<br>" +
+ .setDescription("Returns CE activity related metrics.<br>" +
"Requires 'Administer System' permission or 'Administer' rights on the specified project.")
.setSince("5.5")
.setResponseExample(getClass().getResource("activity_status-example.json"))
.setExampleValue(KeyExamples.KEY_PROJECT_EXAMPLE_001);
action.setChangelog(new Change("6.6", "New field 'inProgress' in response"));
+ action.setChangelog(new Change("7.8", "New field 'pendingTime' in response, only included when there are pending tasks"));
}
@Override
private ActivityStatusWsResponse doHandle(Request request) {
try (DbSession dbSession = dbClient.openSession(false)) {
Optional<ComponentDto> component = searchComponent(dbSession, request);
- String componentUuid = component.isPresent() ? component.get().uuid() : null;
+ String componentUuid = component.map(ComponentDto::uuid).orElse(null);
checkPermissions(component);
int pendingCount = dbClient.ceQueueDao().countByStatusAndMainComponentUuid(dbSession, CeQueueDto.Status.PENDING, componentUuid);
int inProgressCount = dbClient.ceQueueDao().countByStatusAndMainComponentUuid(dbSession, CeQueueDto.Status.IN_PROGRESS, componentUuid);
int failingCount = dbClient.ceActivityDao().countLastByStatusAndMainComponentUuid(dbSession, CeActivityDto.Status.FAILED, componentUuid);
- return ActivityStatusWsResponse.newBuilder()
+ Optional<Long> creationDate = dbClient.ceQueueDao().selectCreationDateOfOldestPendingByMainComponentUuid(dbSession, componentUuid);
+
+ ActivityStatusWsResponse.Builder builder = ActivityStatusWsResponse.newBuilder()
.setPending(pendingCount)
.setInProgress(inProgressCount)
- .setFailing(failingCount)
- .build();
+ .setFailing(failingCount);
+
+ creationDate.ifPresent(d -> {
+ long ageOfOldestPendingTime = system2.now() - d;
+ builder.setPendingTime(ageOfOldestPendingTime);
+ });
+
+ return builder.build();
}
}
if (hasComponentInRequest(request)) {
component = componentFinder.getByUuidOrKey(dbSession, request.getComponentId(), request.getComponentKey(), COMPONENT_ID_AND_KEY);
}
- return Optional.fromNullable(component);
+ return Optional.ofNullable(component);
}
private void checkPermissions(Optional<ComponentDto> component) {
}
private static Request toWsRequest(org.sonar.api.server.ws.Request request) {
- return new Request()
- .setComponentId(request.param(PARAM_COMPONENT_ID))
- .setComponentKey(request.param(DEPRECATED_PARAM_COMPONENT_KEY));
+ return new Request(request.param(PARAM_COMPONENT_ID), request.param(DEPRECATED_PARAM_COMPONENT_KEY));
}
private static class Request {
-
private String componentId;
private String componentKey;
- public Request setComponentId(String componentId) {
+ Request(@Nullable String componentId, @Nullable String componentKey) {
this.componentId = componentId;
- return this;
+ this.componentKey = componentKey;
}
+ @CheckForNull
public String getComponentId() {
return componentId;
}
- public Request setComponentKey(String componentKey) {
- this.componentKey = componentKey;
- return this;
- }
-
+ @CheckForNull
public String getComponentKey() {
return componentKey;
}
{
"pending": 2,
"inProgress": 1,
- "failing": 5
+ "failing": 5,
+ "pendingTime": 100123
}
import org.sonarqube.ws.Ce;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import static org.sonar.db.ce.CeQueueTesting.newCeQueueDto;
import static org.sonar.db.component.ComponentTesting.newPrivateProjectDto;
import static org.sonar.server.ce.ws.CeWsParameters.DEPRECATED_PARAM_COMPONENT_KEY;
@Rule
public DbTester db = DbTester.create(System2.INSTANCE);
+ private System2 system2 = mock(System2.class);
+
private DbClient dbClient = db.getDbClient();
private DbSession dbSession = db.getSession();
- private WsActionTester ws = new WsActionTester(new ActivityStatusAction(userSession, dbClient, TestComponentFinder.from(db)));
+ private WsActionTester ws = new WsActionTester(new ActivityStatusAction(userSession, dbClient, TestComponentFinder.from(db), system2));
@Test
public void test_definition() {
@Test
public void json_example() {
- dbClient.ceQueueDao().insert(dbSession, newCeQueueDto("ce-queue-uuid-1").setStatus(CeQueueDto.Status.PENDING));
+ when(system2.now()).thenReturn(200123L);
+ dbClient.ceQueueDao().insert(dbSession, newCeQueueDto("ce-queue-uuid-1").setStatus(CeQueueDto.Status.PENDING).setCreatedAt(100000));
dbClient.ceQueueDao().insert(dbSession, newCeQueueDto("ce-queue-uuid-2").setStatus(CeQueueDto.Status.PENDING));
dbClient.ceQueueDao().insert(dbSession, newCeQueueDto("ce-queue-uuid-3").setStatus(CeQueueDto.Status.IN_PROGRESS));
for (int i = 0; i < 5; i++) {
assertThat(result.getFailing()).isEqualTo(1);
}
+ @Test
+ public void add_pending_time() {
+ String projectUuid = "project-uuid";
+ OrganizationDto organizationDto = db.organizations().insert();
+ ComponentDto project = newPrivateProjectDto(organizationDto, projectUuid);
+ db.components().insertComponent(project);
+
+ userSession.logIn().addProjectPermission(UserRole.ADMIN, project);
+ when(system2.now()).thenReturn(2000L);
+ insertInQueue(CeQueueDto.Status.PENDING, project, 1000L);
+ Ce.ActivityStatusWsResponse result = call(projectUuid);
+
+ assertThat(result).extracting(Ce.ActivityStatusWsResponse::getPending, Ce.ActivityStatusWsResponse::getFailing,
+ Ce.ActivityStatusWsResponse::getInProgress, Ce.ActivityStatusWsResponse::getPendingTime)
+ .containsOnly(1, 0, 0, 1000L);
+ }
+
@Test
public void empty_status() {
Ce.ActivityStatusWsResponse result = call();
}
private void insertInQueue(CeQueueDto.Status status, @Nullable ComponentDto componentDto) {
- dbClient.ceQueueDao().insert(dbSession, newCeQueueDto(Uuids.createFast())
+ insertInQueue(status, componentDto, null);
+ }
+
+ private void insertInQueue(CeQueueDto.Status status, @Nullable ComponentDto componentDto, @Nullable Long createdAt) {
+ CeQueueDto ceQueueDto = newCeQueueDto(Uuids.createFast())
.setStatus(status)
- .setComponent(componentDto));
+ .setComponent(componentDto);
+ if (createdAt != null) {
+ ceQueueDto.setCreatedAt(createdAt);
+ }
+ dbClient.ceQueueDao().insert(dbSession, ceQueueDto);
db.commit();
}
optional int32 pending = 1;
optional int32 failing = 2;
optional int32 inProgress = 3;
+ optional int64 pendingTime = 4;
}
// GET api/ce/analysis_status
projects / sonarqube.git / commitdiff