SONAR-17699 implemented algorithm for running PRs in parallel

2 files changed, 172 insertions, 128 deletions

diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeQueueDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeQueueDaoTest.java
index 2eb980d3913..b7d473c9e86 100644
--- a/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeQueueDaoTest.java
+++ b/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeQueueDaoTest.java
@@ -39,7 +39,6 @@ import org.sonar.api.impl.utils.AlwaysIncreasingSystem2;
 import org.sonar.api.impl.utils.TestSystem2;
 import org.sonar.api.utils.System2;
 import org.sonar.db.DbTester;
-import org.sonar.db.component.ComponentDto;
 
 import static com.google.common.collect.Lists.newArrayList;
 import static java.util.Collections.singletonList;
@@ -52,6 +51,8 @@ import static org.sonar.db.ce.CeQueueDto.Status.IN_PROGRESS;
 import static org.sonar.db.ce.CeQueueDto.Status.PENDING;
 import static org.sonar.db.ce.CeQueueTesting.newCeQueueDto;
 import static org.sonar.db.ce.CeQueueTesting.reset;
+import static org.sonar.db.ce.CeTaskCharacteristicDto.BRANCH_KEY;
+import static org.sonar.db.ce.CeTaskCharacteristicDto.PULL_REQUEST;
 
 public class CeQueueDaoTest {
   private static final long INIT_TIME = 1_450_000_000_000L;
@@ -59,6 +60,7 @@ public class CeQueueDaoTest {
   private static final String TASK_UUID_2 = "TASK_2";
   private static final String MAIN_COMPONENT_UUID_1 = "PROJECT_1";
   private static final String MAIN_COMPONENT_UUID_2 = "PROJECT_2";
+  private static final String COMPONENT_UUID_1 = "BRANCH_1";
   private static final String TASK_UUID_3 = "TASK_3";
   private static final String SELECT_QUEUE_UUID_AND_STATUS_QUERY = "select uuid,status from ce_queue";
   private static final String SUBMITTER_LOGIN = "submitter uuid";
@@ -392,69 +394,6 @@ public class CeQueueDaoTest {
   }
 
   @Test
-  public void peek_none_if_no_pendings() {
-    assertThat(underTest.peek(db.getSession(), WORKER_UUID_1, false, false)).isNotPresent();
-
-    // not pending, but in progress
-    makeInProgress(WORKER_UUID_1, 2_232_222L, insertPending(TASK_UUID_1, MAIN_COMPONENT_UUID_1));
-    assertThat(underTest.peek(db.getSession(), WORKER_UUID_1, false, false)).isNotPresent();
-  }
-
-  @Test
-  public void peek_oldest_pending() {
-    insertPending(TASK_UUID_1, MAIN_COMPONENT_UUID_1);
-    system2.setNow(INIT_TIME + 3_000_000);
-    insertPending(TASK_UUID_2, MAIN_COMPONENT_UUID_2);
-
-    assertThat(db.countRowsOfTable("ce_queue")).isEqualTo(2);
-    verifyCeQueueStatuses(TASK_UUID_1, PENDING, TASK_UUID_2, PENDING);
-
-    // peek first one
-    Optional<CeQueueDto> peek = underTest.peek(db.getSession(), WORKER_UUID_1, false, false);
-    assertThat(peek).isPresent();
-    assertThat(peek.get().getUuid()).isEqualTo(TASK_UUID_1);
-    assertThat(peek.get().getStatus()).isEqualTo(IN_PROGRESS);
-    assertThat(peek.get().getWorkerUuid()).isEqualTo(WORKER_UUID_1);
-    verifyCeQueueStatuses(TASK_UUID_1, IN_PROGRESS, TASK_UUID_2, PENDING);
-
-    // peek second one
-    peek = underTest.peek(db.getSession(), WORKER_UUID_2, false, false);
-    assertThat(peek).isPresent();
-    assertThat(peek.get().getUuid()).isEqualTo(TASK_UUID_2);
-    assertThat(peek.get().getStatus()).isEqualTo(IN_PROGRESS);
-    assertThat(peek.get().getWorkerUuid()).isEqualTo(WORKER_UUID_2);
-    verifyCeQueueStatuses(TASK_UUID_1, IN_PROGRESS, TASK_UUID_2, IN_PROGRESS);
-
-    // no more pendings
-    assertThat(underTest.peek(db.getSession(), WORKER_UUID_1, false, false)).isNotPresent();
-  }
-
-  @Test
-  public void do_not_peek_multiple_tasks_on_same_main_component_at_the_same_time() {
-    // two pending tasks on the same project
-    insertPending(TASK_UUID_1, MAIN_COMPONENT_UUID_1);
-    system2.setNow(INIT_TIME + 3_000_000);
-    insertPending(TASK_UUID_2, MAIN_COMPONENT_UUID_1);
-
-    Optional<CeQueueDto> peek = underTest.peek(db.getSession(), WORKER_UUID_1, false, false);
-    assertThat(peek).isPresent();
-    assertThat(peek.get().getUuid()).isEqualTo(TASK_UUID_1);
-    assertThat(peek.get().getMainComponentUuid()).isEqualTo(MAIN_COMPONENT_UUID_1);
-    assertThat(peek.get().getWorkerUuid()).isEqualTo(WORKER_UUID_1);
-    verifyCeQueueStatuses(TASK_UUID_1, IN_PROGRESS, TASK_UUID_2, PENDING);
-
-    // do not peek second task as long as the first one is in progress
-    peek = underTest.peek(db.getSession(), WORKER_UUID_1, false, false);
-    assertThat(peek).isEmpty();
-
-    // first one is finished
-    underTest.deleteByUuid(db.getSession(), TASK_UUID_1);
-    peek = underTest.peek(db.getSession(), WORKER_UUID_2, false, false);
-    assertThat(peek.get().getUuid()).isEqualTo(TASK_UUID_2);
-    assertThat(peek.get().getWorkerUuid()).isEqualTo(WORKER_UUID_2);
-  }
-
-  @Test
   public void select_by_query() {
     // task status not in query
     insertPending(newCeQueueDto(TASK_UUID_1)
@@ -632,97 +571,134 @@ public class CeQueueDaoTest {
   }
 
   @Test
-  public void exclude_portfolios_computation_when_indexing_issues() {
-    insertBranch(MAIN_COMPONENT_UUID_1);
+  public void hasAnyIssueSyncTaskPendingOrInProgress_PENDING() {
+    assertThat(underTest.hasAnyIssueSyncTaskPendingOrInProgress(db.getSession())).isFalse();
+
     insertPending(newCeQueueDto(TASK_UUID_1)
       .setComponentUuid(MAIN_COMPONENT_UUID_1)
-      .setMainComponentUuid(MAIN_COMPONENT_UUID_1)
       .setStatus(PENDING)
       .setTaskType(CeTaskTypes.BRANCH_ISSUE_SYNC)
       .setCreatedAt(100_000L));
 
-    String view_uuid = "view_uuid";
-    insertView(view_uuid);
-    insertPending(newCeQueueDto(TASK_UUID_2)
-      .setComponentUuid(view_uuid)
-      .setMainComponentUuid(view_uuid)
-      .setStatus(PENDING)
-      .setTaskType(CeTaskTypes.REPORT)
-      .setCreatedAt(100_000L));
-
-    Optional<CeQueueDto> peek = underTest.peek(db.getSession(), WORKER_UUID_1, false, true);
-    assertThat(peek).isPresent();
-    assertThat(peek.get().getUuid()).isEqualTo(TASK_UUID_1);
-
-    Optional<CeQueueDto> peek2 = underTest.peek(db.getSession(), WORKER_UUID_1, false, false);
-    assertThat(peek2).isPresent();
-    assertThat(peek2.get().getUuid()).isEqualTo(TASK_UUID_2);
+    assertThat(underTest.hasAnyIssueSyncTaskPendingOrInProgress(db.getSession())).isTrue();
   }
 
   @Test
-  public void excluding_view_pick_up_orphan_branches() {
+  public void hasAnyIssueSyncTaskPendingOrInProgress_IN_PROGRESS() {
+    assertThat(underTest.hasAnyIssueSyncTaskPendingOrInProgress(db.getSession())).isFalse();
+
     insertPending(newCeQueueDto(TASK_UUID_1)
       .setComponentUuid(MAIN_COMPONENT_UUID_1)
-      .setMainComponentUuid("non-existing-uuid")
-      .setStatus(PENDING)
+      .setStatus(IN_PROGRESS)
       .setTaskType(CeTaskTypes.BRANCH_ISSUE_SYNC)
       .setCreatedAt(100_000L));
 
-    Optional<CeQueueDto> peek = underTest.peek(db.getSession(), WORKER_UUID_1, false, true);
-    assertThat(peek).isPresent();
-    assertThat(peek.get().getUuid()).isEqualTo(TASK_UUID_1);
+    assertThat(underTest.hasAnyIssueSyncTaskPendingOrInProgress(db.getSession())).isTrue();
   }
 
   @Test
-  public void hasAnyIssueSyncTaskPendingOrInProgress_PENDING() {
-    assertThat(underTest.hasAnyIssueSyncTaskPendingOrInProgress(db.getSession())).isFalse();
+  public void selectOldestPendingPrOrBranch_returns_oldest_100_pr_or_branch_tasks() {
+    for (int i = 1; i < 110; i++) {
+      insertPending(newCeQueueDto("task" + i)
+        .setComponentUuid(MAIN_COMPONENT_UUID_1).setStatus(PENDING).setTaskType(CeTaskTypes.REPORT).setCreatedAt(i));
+    }
+    for (int i = 1; i < 10; i++) {
+      insertPending(newCeQueueDto("progress" + i)
+        .setComponentUuid(MAIN_COMPONENT_UUID_1).setStatus(IN_PROGRESS).setTaskType(CeTaskTypes.REPORT).setCreatedAt(i));
+      insertPending(newCeQueueDto("sync" + i)
+        .setComponentUuid(MAIN_COMPONENT_UUID_1).setStatus(PENDING).setTaskType(CeTaskTypes.BRANCH_ISSUE_SYNC).setCreatedAt(i));
+    }
+
+    List<PrOrBranchTask> prOrBranchTasks = underTest.selectOldestPendingPrOrBranch(db.getSession());
+    assertThat(prOrBranchTasks).hasSize(100)
+      .allMatch(t -> t.getCeTaskUuid().startsWith("task"), "uuid starts with task")
+      .allMatch(t -> t.getCreatedAt() <= 100, "creation date older or equal than 100");
+  }
+
+  @Test
+  public void selectOldestPendingPrOrBranch_returns_branch_branch_type_if_no_characteristics() {
+    insertPending(newCeQueueDto(TASK_UUID_1)
+      .setComponentUuid(COMPONENT_UUID_1)
+      .setMainComponentUuid(MAIN_COMPONENT_UUID_1)
+      .setStatus(PENDING)
+      .setTaskType(CeTaskTypes.REPORT)
+      .setCreatedAt(123L));
+    List<PrOrBranchTask> prOrBranchTasks = underTest.selectOldestPendingPrOrBranch(db.getSession());
 
+    assertThat(prOrBranchTasks).hasSize(1);
+    assertThat(prOrBranchTasks.get(0))
+      .extracting(PrOrBranchTask::getBranchType, PrOrBranchTask::getComponentUuid, PrOrBranchTask::getMainComponentUuid, PrOrBranchTask::getTaskType)
+      .containsExactly(BRANCH_KEY, COMPONENT_UUID_1, MAIN_COMPONENT_UUID_1, CeTaskTypes.REPORT);
+  }
+
+  @Test
+  public void selectOldestPendingPrOrBranch_returns_branch_branch_type_if_unrelated_characteristics() {
     insertPending(newCeQueueDto(TASK_UUID_1)
-      .setComponentUuid(MAIN_COMPONENT_UUID_1)
+      .setComponentUuid(COMPONENT_UUID_1)
+      .setMainComponentUuid(MAIN_COMPONENT_UUID_1)
       .setStatus(PENDING)
-      .setTaskType(CeTaskTypes.BRANCH_ISSUE_SYNC)
-      .setCreatedAt(100_000L));
+      .setTaskType(CeTaskTypes.REPORT)
+      .setCreatedAt(123L));
+    List<PrOrBranchTask> prOrBranchTasks = underTest.selectOldestPendingPrOrBranch(db.getSession());
+    insertCharacteristic(BRANCH_KEY, "123", "c1", TASK_UUID_1);
 
-    assertThat(underTest.hasAnyIssueSyncTaskPendingOrInProgress(db.getSession())).isTrue();
+    assertThat(prOrBranchTasks).hasSize(1);
+    assertThat(prOrBranchTasks.get(0))
+      .extracting(PrOrBranchTask::getBranchType, PrOrBranchTask::getComponentUuid, PrOrBranchTask::getMainComponentUuid, PrOrBranchTask::getTaskType)
+      .containsExactly(BRANCH_KEY, COMPONENT_UUID_1, MAIN_COMPONENT_UUID_1, CeTaskTypes.REPORT);
   }
 
   @Test
-  public void hasAnyIssueSyncTaskPendingOrInProgress_IN_PROGRESS() {
-    assertThat(underTest.hasAnyIssueSyncTaskPendingOrInProgress(db.getSession())).isFalse();
+  public void selectOldestPendingPrOrBranch_returns_all_fields() {
+    insertPending(newCeQueueDto(TASK_UUID_1)
+      .setComponentUuid(COMPONENT_UUID_1)
+      .setMainComponentUuid(MAIN_COMPONENT_UUID_1)
+      .setStatus(PENDING)
+      .setTaskType(CeTaskTypes.REPORT)
+      .setCreatedAt(123L));
+    insertCharacteristic(PULL_REQUEST, "1", "c1", TASK_UUID_1);
+
+    List<PrOrBranchTask> prOrBranchTasks = underTest.selectOldestPendingPrOrBranch(db.getSession());
+
+    assertThat(prOrBranchTasks).hasSize(1);
+    assertThat(prOrBranchTasks.get(0))
+      .extracting(PrOrBranchTask::getBranchType, PrOrBranchTask::getComponentUuid, PrOrBranchTask::getMainComponentUuid, PrOrBranchTask::getTaskType)
+      .containsExactly(PULL_REQUEST, COMPONENT_UUID_1, MAIN_COMPONENT_UUID_1, CeTaskTypes.REPORT);
+  }
 
+  @Test
+  public void selectInProgressWithCharacteristics_returns_all_fields() {
     insertPending(newCeQueueDto(TASK_UUID_1)
-      .setComponentUuid(MAIN_COMPONENT_UUID_1)
+      .setComponentUuid(COMPONENT_UUID_1)
+      .setMainComponentUuid(MAIN_COMPONENT_UUID_1)
       .setStatus(IN_PROGRESS)
-      .setTaskType(CeTaskTypes.BRANCH_ISSUE_SYNC)
-      .setCreatedAt(100_000L));
+      .setTaskType(CeTaskTypes.REPORT)
+      .setCreatedAt(123L));
+    insertCharacteristic(PULL_REQUEST, "1", "c1", TASK_UUID_1);
 
-    assertThat(underTest.hasAnyIssueSyncTaskPendingOrInProgress(db.getSession())).isTrue();
-  }
+    List<PrOrBranchTask> prOrBranchTasks = underTest.selectInProgressWithCharacteristics(db.getSession());
 
-  private void insertView(String view_uuid) {
-    ComponentDto view = new ComponentDto();
-    view.setQualifier("VW");
-    view.setKey(view_uuid + "_key");
-    view.setUuid(view_uuid);
-    view.setPrivate(false);
-    view.setRootUuid(view_uuid);
-    view.setUuidPath("uuid_path");
-    view.setBranchUuid(view_uuid);
-    db.components().insertPortfolioAndSnapshot(view);
-    db.commit();
+    assertThat(prOrBranchTasks).hasSize(1);
+    assertThat(prOrBranchTasks.get(0))
+      .extracting(PrOrBranchTask::getBranchType, PrOrBranchTask::getComponentUuid, PrOrBranchTask::getMainComponentUuid, PrOrBranchTask::getTaskType)
+      .containsExactly(PULL_REQUEST, COMPONENT_UUID_1, MAIN_COMPONENT_UUID_1, CeTaskTypes.REPORT);
   }
 
-  private void insertBranch(String uuid) {
-    ComponentDto branch = new ComponentDto();
-    branch.setQualifier("TRK");
-    branch.setKey(uuid + "_key");
-    branch.setUuid(uuid);
-    branch.setPrivate(false);
-    branch.setRootUuid(uuid);
-    branch.setUuidPath("uuid_path");
-    branch.setBranchUuid(uuid);
-    db.components().insertComponent(branch);
-    db.commit();
+  @Test
+  public void selectInProgressWithCharacteristics_returns_branch_branch_type_if_no_characteristics() {
+    insertPending(newCeQueueDto(TASK_UUID_1)
+      .setComponentUuid(COMPONENT_UUID_1)
+      .setMainComponentUuid(MAIN_COMPONENT_UUID_1)
+      .setStatus(IN_PROGRESS)
+      .setTaskType(CeTaskTypes.REPORT)
+      .setCreatedAt(123L));
+
+    List<PrOrBranchTask> prOrBranchTasks = underTest.selectInProgressWithCharacteristics(db.getSession());
+
+    assertThat(prOrBranchTasks).hasSize(1);
+    assertThat(prOrBranchTasks.get(0))
+      .extracting(PrOrBranchTask::getBranchType, PrOrBranchTask::getComponentUuid, PrOrBranchTask::getMainComponentUuid, PrOrBranchTask::getTaskType)
+      .containsExactly(BRANCH_KEY, COMPONENT_UUID_1, MAIN_COMPONENT_UUID_1, CeTaskTypes.REPORT);
   }
 
   private void insertPending(CeQueueDto dto) {
@@ -769,6 +745,15 @@ public class CeQueueDaoTest {
     return underTest.selectByUuid(db.getSession(), uuid).get();
   }
 
+  private void insertCharacteristic(String key, String value, String uuid, String taskUuid) {
+    CeTaskCharacteristicDto dto1 = new CeTaskCharacteristicDto()
+      .setKey(key)
+      .setValue(value)
+      .setUuid(uuid)
+      .setTaskUuid(taskUuid);
+    db.getDbClient().ceTaskCharacteristicsDao().insert(db.getSession(), dto1);
+  }
+
   private static Iterable<Map<String, Object>> upperizeKeys(List<Map<String, Object>> select) {
     return select.stream().map(new Function<Map<String, Object>, Map<String, Object>>() {
       @Nullable
@@ -783,10 +768,6 @@ public class CeQueueDaoTest {
     }).collect(Collectors.toList());
   }
 
-  private void verifyCeQueueStatuses(String taskUuid1, CeQueueDto.Status taskStatus1, String taskUuid2, CeQueueDto.Status taskStatus2) {
-    verifyCeQueueStatuses(new String[] {taskUuid1, taskUuid2}, new CeQueueDto.Status[] {taskStatus1, taskStatus2});
-  }
-
   private void verifyCeQueueStatuses(String[] taskUuids, CeQueueDto.Status[] statuses) {
     Map<String, Object>[] rows = new Map[taskUuids.length];
     for (int i = 0; i < taskUuids.length; i++) {
diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeTaskDtoLightTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeTaskDtoLightTest.java
new file mode 100644
index 00000000000..1691738deaf
--- /dev/null
+++ b/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeTaskDtoLightTest.java
@@ -0,0 +1,63 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.db.ce;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class CeTaskDtoLightTest {
+  private final CeTaskDtoLight task1 = new CeTaskDtoLight();
+  private final CeTaskDtoLight task2 = new CeTaskDtoLight();
+  private final CeTaskDtoLight task3 = new CeTaskDtoLight();
+  private final CeTaskDtoLight task4 = new CeTaskDtoLight();
+
+  @Before
+  public void setUp() {
+    task1.setCeTaskUuid("id1");
+    task1.setCreatedAt(1);
+    task2.setCeTaskUuid("id1");
+    task2.setCreatedAt(1);
+    task3.setCeTaskUuid("id2");
+    task3.setCreatedAt(1);
+    task4.setCeTaskUuid("id1");
+    task4.setCreatedAt(2);
+  }
+
+  @Test
+  public void equals_is_based_on_created_date_and_uuid() {
+    assertThat(task1).isEqualTo(task2);
+    assertThat(task1).isNotEqualTo(task3);
+    assertThat(task1).isNotEqualTo(task4);
+  }
+
+  @Test
+  public void hashCode_is_based_on_created_date_and_uuid() {
+    assertThat(task1).hasSameHashCodeAs(task2);
+  }
+
+  @Test
+  public void compareTo_is_based_on_created_date_and_uuid() {
+    assertThat(task1).isEqualByComparingTo(task2);
+    assertThat(task1).isLessThan(task3);
+    assertThat(task1).isLessThan(task4);
+  }
+}