diff options
14 files changed, 220 insertions, 27 deletions
diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeActivityDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeActivityDao.java index af9b1e4667f..1c174adef03 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeActivityDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeActivityDao.java @@ -88,6 +88,10 @@ public class CeActivityDao implements Dao { return Optional.ofNullable(mapper(dbSession).selectLastByComponentUuidAndTaskType(componentUuid, taskType)); } + public boolean hasAnyFailedIssueSyncTask(DbSession dbSession) { + return mapper(dbSession).hasAnyFailedIssueSyncTask() > 0; + } + private static CeActivityMapper mapper(DbSession dbSession) { return dbSession.getMapper(CeActivityMapper.class); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeQueueDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeQueueDao.java index 3e699d894b7..8b396644d2b 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeQueueDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeQueueDao.java @@ -190,6 +190,10 @@ public class CeQueueDao implements Dao { return Optional.ofNullable(result); } + public boolean hasAnyIssueSyncTaskPendingOrInProgress(DbSession dbSession) { + return mapper(dbSession).hasAnyIssueSyncTaskPendingOrInProgress() > 0; + } + private static CeQueueMapper mapper(DbSession session) { return session.getMapper(CeQueueMapper.class); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeQueueMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeQueueMapper.java index fd277b2011c..8483d81a109 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeQueueMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeQueueMapper.java @@ -84,4 +84,7 @@ public interface CeQueueMapper { @Param("old") UpdateIf.OldProperties oldProperties); int deleteByUuid(@Param("uuid") String uuid, @Nullable @Param("deleteIf") DeleteIf deleteIf); + + short hasAnyIssueSyncTaskPendingOrInProgress(); + } diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeActivityMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeActivityMapper.xml index 1beca1ae98d..1cf33d044f3 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeActivityMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeActivityMapper.xml @@ -238,14 +238,26 @@ <select id="selectLastByComponentUuidAndTaskType" parameterType="map" resultType="org.sonar.db.ce.CeActivityDto"> select - <include refid="columns"/> + <include refid="columns"/> from ce_activity ca left outer join ce_scanner_context csc on - ca.uuid = csc.task_uuid + ca.uuid = csc.task_uuid where - ca.component_uuid = #{componentUuid,jdbcType=VARCHAR} - and ca.task_type = #{taskType,jdbcType=VARCHAR} - and ca.is_last = ${_true} + ca.component_uuid = #{componentUuid,jdbcType=VARCHAR} + and ca.task_type = #{taskType,jdbcType=VARCHAR} + and ca.is_last = ${_true} + </select> + + <select id="hasAnyFailedIssueSyncTask" parameterType="map" resultType="short"> + select + case when exists + ( + select ca.uuid from ce_activity ca where ca.task_type = 'ISSUE_SYNC' and status = 'FAILED' + and exists(select pb.uuid from project_branches pb where ca.component_uuid = pb.uuid and pb.need_issue_sync = ${_true}) + ) + then 1 + else 0 + end </select> <select id="selectByTaskType" parameterType="map" resultType="org.sonar.db.ce.CeActivityDto"> diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeQueueMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeQueueMapper.xml index 5ac5844792b..d4b115258dc 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeQueueMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeQueueMapper.xml @@ -319,4 +319,16 @@ where status = 'IN_PROGRESS' </update> + + <select id="hasAnyIssueSyncTaskPendingOrInProgress" parameterType="map" resultType="short"> + select + case when exists + ( + select cq.uuid from ce_queue cq where cq.task_type = 'ISSUE_SYNC' and status in ('IN_PROGRESS', 'PENDING') + ) + then 1 + else 0 + end + </select> + </mapper> diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeActivityDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeActivityDaoTest.java index 0e10b7b101b..79bf805add8 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeActivityDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeActivityDaoTest.java @@ -47,6 +47,8 @@ import org.sonar.core.util.stream.MoreCollectors; import org.sonar.db.DbSession; import org.sonar.db.DbTester; import org.sonar.db.Pagination; +import org.sonar.db.component.BranchDto; +import org.sonar.db.project.ProjectDto; import static java.util.Collections.singleton; import static java.util.Collections.singletonList; diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeQueueDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeQueueDaoTest.java index 83fdb26f1dc..a74006db480 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeQueueDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeQueueDaoTest.java @@ -659,6 +659,32 @@ public class CeQueueDaoTest { assertThat(peek2.get().getUuid()).isEqualTo(TASK_UUID_2); } + @Test + public void hasAnyIssueSyncTaskPendingOrInProgress_PENDING() { + assertThat(underTest.hasAnyIssueSyncTaskPendingOrInProgress(db.getSession())).isFalse(); + + insertPending(newCeQueueDto(TASK_UUID_1) + .setComponentUuid(MAIN_COMPONENT_UUID_1) + .setStatus(PENDING) + .setTaskType(CeTaskTypes.BRANCH_ISSUE_SYNC) + .setCreatedAt(100_000L)); + + assertThat(underTest.hasAnyIssueSyncTaskPendingOrInProgress(db.getSession())).isTrue(); + } + + @Test + public void hasAnyIssueSyncTaskPendingOrInProgress_IN_PROGRESS() { + assertThat(underTest.hasAnyIssueSyncTaskPendingOrInProgress(db.getSession())).isFalse(); + + insertPending(newCeQueueDto(TASK_UUID_1) + .setComponentUuid(MAIN_COMPONENT_UUID_1) + .setStatus(IN_PROGRESS) + .setTaskType(CeTaskTypes.BRANCH_ISSUE_SYNC) + .setCreatedAt(100_000L)); + + assertThat(underTest.hasAnyIssueSyncTaskPendingOrInProgress(db.getSession())).isTrue(); + } + private void insertView(String view_uuid) { ComponentDto view = new ComponentDto(); view.setQualifier("VW"); diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndexSyncProgressChecker.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndexSyncProgressChecker.java index 8af309c34cf..04dddb7da78 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndexSyncProgressChecker.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndexSyncProgressChecker.java @@ -36,8 +36,10 @@ public class IssueIndexSyncProgressChecker { public IssueSyncProgress getIssueSyncProgress(DbSession dbSession) { int completed = dbClient.branchDao().countByNeedIssueSync(dbSession, false); + boolean hasFailures = dbClient.ceActivityDao().hasAnyFailedIssueSyncTask(dbSession); + boolean isCompleted = !dbClient.ceQueueDao().hasAnyIssueSyncTaskPendingOrInProgress(dbSession); int total = dbClient.branchDao().countAll(dbSession); - return new IssueSyncProgress(completed, total); + return new IssueSyncProgress(isCompleted, completed, total, hasFailures); } public void checkIfAnyComponentsNeedIssueSync(DbSession dbSession, List<String> componentKeys) { diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueSyncProgress.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueSyncProgress.java index 193db999a9c..84f9bc9895d 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueSyncProgress.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueSyncProgress.java @@ -25,8 +25,13 @@ public class IssueSyncProgress { private final int completed; private final int total; - public IssueSyncProgress(int completed, int total) { + private final boolean hasFailures; + private final boolean isCompleted; + + public IssueSyncProgress(boolean isCompleted, int completed, int total, boolean hasFailures) { this.completed = completed; + this.hasFailures = hasFailures; + this.isCompleted = isCompleted; this.total = total; } @@ -34,6 +39,10 @@ public class IssueSyncProgress { return completed; } + public boolean hasFailures() { + return hasFailures; + } + public int getTotal() { return total; } @@ -46,6 +55,6 @@ public class IssueSyncProgress { } public boolean isCompleted() { - return completed == total; + return completed == total || isCompleted; } } diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSyncProgressCheckerTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSyncProgressCheckerTest.java index 7ef82fa5ad0..254d2f58ba5 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSyncProgressCheckerTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSyncProgressCheckerTest.java @@ -32,6 +32,10 @@ import org.junit.runner.RunWith; import org.sonar.api.utils.System2; import org.sonar.db.DbSession; import org.sonar.db.DbTester; +import org.sonar.db.ce.CeActivityDto; +import org.sonar.db.ce.CeQueueDto; +import org.sonar.db.ce.CeQueueDto.Status; +import org.sonar.db.ce.CeTaskTypes; import org.sonar.db.component.ComponentDto; import org.sonar.db.project.ProjectDto; import org.sonar.server.es.EsIndexSyncInProgressException; @@ -39,10 +43,14 @@ import org.sonar.server.es.EsIndexSyncInProgressException; import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.sonar.db.ce.CeActivityDto.Status.FAILED; +import static org.sonar.db.ce.CeActivityDto.Status.SUCCESS; @RunWith(DataProviderRunner.class) public class IssueIndexSyncProgressCheckerTest { + private System2 system2 = new System2(); + @Rule public DbTester db = DbTester.create(System2.INSTANCE); @@ -55,6 +63,7 @@ public class IssueIndexSyncProgressCheckerTest { assertThat(issueSyncProgress.getTotal()).isZero(); assertThat(issueSyncProgress.toPercentCompleted()).isEqualTo(100); assertThat(issueSyncProgress.isCompleted()).isTrue(); + assertThat(issueSyncProgress.hasFailures()).isFalse(); } @Test @@ -71,8 +80,26 @@ public class IssueIndexSyncProgressCheckerTest { } @Test + public void return_has_failure_true_if_exists_task() { + assertThat(underTest.getIssueSyncProgress(db.getSession()).hasFailures()).isFalse(); + + ProjectDto projectDto1 = insertProjectWithBranches(false, 0); + insertCeActivity("TASK_1", projectDto1, SUCCESS); + + ProjectDto projectDto2 = insertProjectWithBranches(false, 0); + insertCeActivity("TASK_2", projectDto2, SUCCESS); + + assertThat(underTest.getIssueSyncProgress(db.getSession()).hasFailures()).isFalse(); + + ProjectDto projectDto3 = insertProjectWithBranches(true, 0); + insertCeActivity("TASK_3", projectDto3, FAILED); + + assertThat(underTest.getIssueSyncProgress(db.getSession()).hasFailures()).isTrue(); + } + + @Test @UseDataProvider("various_task_numbers") - public void return_correct_percent_value_for_branches_to_sync(int toSync, int synced, int expectedPercent, boolean isCompleted) { + public void return_correct_percent_value_for_branches_to_sync(int toSync, int synced, int expectedPercent) { IntStream.range(0, toSync).forEach(value -> insertProjectWithBranches(true, 0)); IntStream.range(0, synced).forEach(value -> insertProjectWithBranches(false, 0)); @@ -80,28 +107,27 @@ public class IssueIndexSyncProgressCheckerTest { assertThat(result.getCompleted()).isEqualTo(synced); assertThat(result.getTotal()).isEqualTo(toSync + synced); assertThat(result.toPercentCompleted()).isEqualTo(expectedPercent); - assertThat(result.isCompleted()).isEqualTo(isCompleted); } @DataProvider public static Object[][] various_task_numbers() { return new Object[][] { - // toSync, synced, expected result, expectedCompleted - {0, 0, 100, true}, - {0, 9, 100, true}, - {10, 0, 0, false}, - {99, 1, 1, false}, - {2, 1, 33, false}, - {6, 4, 40, false}, - {7, 7, 50, false}, - {1, 2, 66, false}, - {4, 10, 71, false}, - {1, 99, 99, false}, + // toSync, synced, expected result + {0, 0, 100}, + {0, 9, 100}, + {10, 0, 0}, + {99, 1, 1}, + {2, 1, 33}, + {6, 4, 40}, + {7, 7, 50}, + {1, 2, 66}, + {4, 10, 71}, + {1, 99, 99}, }; } @Test - public void return_0_if_all_branches_have_need_issue_sync_set_TRUE() { + public void return_0_if_all_branches_have_need_issue_sync_set_true() { // only project IntStream.range(0, 10).forEach(value -> insertProjectWithBranches(true, 0)); @@ -112,6 +138,69 @@ public class IssueIndexSyncProgressCheckerTest { assertThat(result.getCompleted()).isZero(); assertThat(result.getTotal()).isEqualTo(30); assertThat(result.toPercentCompleted()).isZero(); + } + + @Test + public void return_is_completed_true_if_no_pending_or_in_progress_tasks() { + // only project + IntStream.range(0, 10).forEach(value -> insertProjectWithBranches(false, 0)); + + // project + additional branch + IntStream.range(0, 10).forEach(value -> insertProjectWithBranches(false, 1)); + + IssueSyncProgress result = underTest.getIssueSyncProgress(db.getSession()); + assertThat(result.isCompleted()).isTrue(); + } + + @Test + public void return_is_completed_true_if_pending_task_exist_but_all_branches_have_been_synced() { + insertCeQueue("TASK_1", Status.PENDING); + // only project + IntStream.range(0, 10).forEach(value -> insertProjectWithBranches(false, 0)); + + // project + additional branch + IntStream.range(0, 10).forEach(value -> insertProjectWithBranches(false, 1)); + + IssueSyncProgress result = underTest.getIssueSyncProgress(db.getSession()); + assertThat(result.isCompleted()).isTrue(); + } + + @Test + public void return_is_completed_true_if_in_progress_task_exist_but_all_branches_have_been_synced() { + insertCeQueue("TASK_1", Status.IN_PROGRESS); + // only project + IntStream.range(0, 10).forEach(value -> insertProjectWithBranches(false, 0)); + + // project + additional branch + IntStream.range(0, 10).forEach(value -> insertProjectWithBranches(false, 1)); + + IssueSyncProgress result = underTest.getIssueSyncProgress(db.getSession()); + assertThat(result.isCompleted()).isTrue(); + } + + @Test + public void return_is_completed_false_if_pending_task_exist_and_branches_need_issue_sync() { + insertCeQueue("TASK_1", Status.PENDING); + // only project + IntStream.range(0, 10).forEach(value -> insertProjectWithBranches(true, 0)); + + // project + additional branch + IntStream.range(0, 10).forEach(value -> insertProjectWithBranches(false, 1)); + + IssueSyncProgress result = underTest.getIssueSyncProgress(db.getSession()); + assertThat(result.isCompleted()).isFalse(); + } + + @Test + public void return_is_completed_false_if_in_progress_task_exist_and_branches_need_issue_sync() { + insertCeQueue("TASK_1", Status.IN_PROGRESS); + // only project + IntStream.range(0, 10).forEach(value -> insertProjectWithBranches(true, 0)); + + // project + additional branch + IntStream.range(0, 10).forEach(value -> insertProjectWithBranches(false, 1)); + + IssueSyncProgress result = underTest.getIssueSyncProgress(db.getSession()); assertThat(result.isCompleted()).isFalse(); } @@ -161,7 +250,7 @@ public class IssueIndexSyncProgressCheckerTest { List<String> projectKey2 = singletonList(projectDto1.getKey()); // throws if flag set to TRUE assertThatThrownBy(() -> underTest.checkIfAnyComponentsNeedIssueSync(session, - projectKey2)) + projectKey2)) .isInstanceOf(EsIndexSyncInProgressException.class) .hasFieldOrPropertyWithValue("httpCode", 503) .hasMessage("Results are temporarily unavailable. Indexing of issues is in progress."); @@ -226,4 +315,29 @@ public class IssueIndexSyncProgressCheckerTest { i -> db.components().insertProjectBranch(projectDto, branchDto -> branchDto.setNeedIssueSync(needIssueSync))); return projectDto; } + + private CeQueueDto insertCeQueue(String uuid, CeQueueDto.Status status) { + CeQueueDto queueDto = new CeQueueDto(); + queueDto.setUuid(uuid); + queueDto.setStatus(status); + queueDto.setTaskType(CeTaskTypes.BRANCH_ISSUE_SYNC); + db.getDbClient().ceQueueDao().insert(db.getSession(), queueDto); + return queueDto; + } + + private CeActivityDto insertCeActivity(String uuid, ProjectDto projectDto, CeActivityDto.Status status) { + CeQueueDto queueDto = new CeQueueDto(); + queueDto.setUuid(uuid); + queueDto.setTaskType(CeTaskTypes.BRANCH_ISSUE_SYNC); + + CeActivityDto dto = new CeActivityDto(queueDto); + dto.setComponentUuid(projectDto.getUuid()); + dto.setMainComponentUuid(projectDto.getUuid()); + dto.setStatus(status); + dto.setTaskType(CeTaskTypes.BRANCH_ISSUE_SYNC); + dto.setAnalysisUuid(uuid + "_AA"); + dto.setCreatedAt(system2.now()); + db.getDbClient().ceActivityDao().insert(db.getSession(), dto); + return dto; + } } diff --git a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/ce/ws/IndexationStatusAction.java b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/ce/ws/IndexationStatusAction.java index 723833d1bc5..f4769816e78 100644 --- a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/ce/ws/IndexationStatusAction.java +++ b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/ce/ws/IndexationStatusAction.java @@ -64,6 +64,7 @@ public class IndexationStatusAction implements CeWsAction { return IndexationStatusWsResponse.newBuilder() .setIsCompleted(issueSyncProgress.isCompleted()) .setPercentCompleted(issueSyncProgress.toPercentCompleted()) + .setHasFailures(issueSyncProgress.hasFailures()) .build(); } diff --git a/server/sonar-webserver-webapi/src/main/resources/org/sonar/server/ce/ws/indexation_status-example.json b/server/sonar-webserver-webapi/src/main/resources/org/sonar/server/ce/ws/indexation_status-example.json index 16ddbfdd709..a2a31b63a19 100644 --- a/server/sonar-webserver-webapi/src/main/resources/org/sonar/server/ce/ws/indexation_status-example.json +++ b/server/sonar-webserver-webapi/src/main/resources/org/sonar/server/ce/ws/indexation_status-example.json @@ -1,4 +1,5 @@ { "isCompleted": true, - "percentCompleted": 100 + "percentCompleted": 100, + "hasFailures": false } diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/ce/ws/IndexationStatusActionTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/ce/ws/IndexationStatusActionTest.java index f5a66a0995a..d6493982d2f 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/ce/ws/IndexationStatusActionTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/ce/ws/IndexationStatusActionTest.java @@ -61,26 +61,28 @@ public class IndexationStatusActionTest { @Test public void verify_example_of_response() { - when(issueIndexSyncProgressCheckerMock.getIssueSyncProgress(any())).thenReturn(new IssueSyncProgress(0, 0)); + when(issueIndexSyncProgressCheckerMock.getIssueSyncProgress(any())).thenReturn(new IssueSyncProgress(true,0, 0, false)); ws.newRequest().execute().assertJson(ws.getDef().responseExampleAsString()); } @Test public void return_100_if_there_is_no_tasks_left() { - when(issueIndexSyncProgressCheckerMock.getIssueSyncProgress(any())).thenReturn(new IssueSyncProgress(10, 10)); + when(issueIndexSyncProgressCheckerMock.getIssueSyncProgress(any())).thenReturn(new IssueSyncProgress(true, 10, 10, false)); IndexationStatusWsResponse response = ws.newRequest() .executeProtobuf(IndexationStatusWsResponse.class); assertThat(response.getPercentCompleted()).isEqualTo(100); assertThat(response.getIsCompleted()).isTrue(); + assertThat(response.getHasFailures()).isFalse(); } @Test public void return_0_if_all_branches_have_need_issue_sync_set_TRUE() { - when(issueIndexSyncProgressCheckerMock.getIssueSyncProgress(any())).thenReturn(new IssueSyncProgress(0, 10)); + when(issueIndexSyncProgressCheckerMock.getIssueSyncProgress(any())).thenReturn(new IssueSyncProgress(false,0, 10, false)); IndexationStatusWsResponse response = ws.newRequest() .executeProtobuf(IndexationStatusWsResponse.class); assertThat(response.getPercentCompleted()).isZero(); assertThat(response.getIsCompleted()).isFalse(); + assertThat(response.getHasFailures()).isFalse(); } } diff --git a/sonar-ws/src/main/protobuf/ws-ce.proto b/sonar-ws/src/main/protobuf/ws-ce.proto index a387200b138..b7938c61d53 100644 --- a/sonar-ws/src/main/protobuf/ws-ce.proto +++ b/sonar-ws/src/main/protobuf/ws-ce.proto @@ -56,6 +56,7 @@ message ActivityStatusWsResponse { message IndexationStatusWsResponse { optional bool isCompleted = 1; optional int32 percentCompleted = 2; + optional bool hasFailures = 3; } // GET api/ce/analysis_status |