SONAR-9973 IT for report processing resilient to failure during start

of components

5 files changed, 166 insertions, 34 deletions

diff --git a/tests/plugins/fake-governance-plugin/src/main/java/ce/BombConfig.java b/tests/plugins/fake-governance-plugin/src/main/java/ce/BombConfig.java
index 52207065d1a..3e5af086b86 100644
--- a/tests/plugins/fake-governance-plugin/src/main/java/ce/BombConfig.java
+++ b/tests/plugins/fake-governance-plugin/src/main/java/ce/BombConfig.java
@@ -27,6 +27,8 @@ import org.sonar.db.DbSession;
 @ServerSide
 @ComputeEngineSide
 public class BombConfig {
+  private static final String OOM_START_BOMB_KEY = "oomStartBomb";
+  private static final String ISE_START_BOMB_KEY = "iseStartBomb";
   private static final String OOM_STOP_BOMB_KEY = "oomStopBomb";
   private static final String ISE_STOP_BOMB_KEY = "iseStopBomb";
 
@@ -36,6 +38,42 @@ public class BombConfig {
     this.dbClient = dbClient;
   }
 
+  public void reset() {
+    try (DbSession dbSession = dbClient.openSession(false)) {
+      dbClient.internalPropertiesDao().save(dbSession, OOM_START_BOMB_KEY, String.valueOf(false));
+      dbClient.internalPropertiesDao().save(dbSession, ISE_START_BOMB_KEY, String.valueOf(false));
+      dbClient.internalPropertiesDao().save(dbSession, OOM_STOP_BOMB_KEY, String.valueOf(false));
+      dbClient.internalPropertiesDao().save(dbSession, ISE_STOP_BOMB_KEY, String.valueOf(false));
+      dbSession.commit();
+    }
+  }
+
+  public boolean isOomStartBomb() {
+    try (DbSession dbSession = dbClient.openSession(false)) {
+      return dbClient.internalPropertiesDao().selectByKey(dbSession, OOM_START_BOMB_KEY).map(Boolean::valueOf).orElse(false);
+    }
+  }
+
+  public void setOomStartBomb(boolean oomStartBomb) {
+    try (DbSession dbSession = dbClient.openSession(false)) {
+      dbClient.internalPropertiesDao().save(dbSession, OOM_START_BOMB_KEY, String.valueOf(oomStartBomb));
+      dbSession.commit();
+    }
+  }
+
+  public boolean isIseStartBomb() {
+    try (DbSession dbSession = dbClient.openSession(false)) {
+      return dbClient.internalPropertiesDao().selectByKey(dbSession, ISE_START_BOMB_KEY).map(Boolean::valueOf).orElse(false);
+    }
+  }
+
+  public void setIseStartBomb(boolean iseStartBomb) {
+    try (DbSession dbSession = dbClient.openSession(false)) {
+      dbClient.internalPropertiesDao().save(dbSession, ISE_START_BOMB_KEY, String.valueOf(iseStartBomb));
+      dbSession.commit();
+    }
+  }
+
   public boolean isOomStopBomb() {
     try (DbSession dbSession = dbClient.openSession(false)) {
       return dbClient.internalPropertiesDao().selectByKey(dbSession, OOM_STOP_BOMB_KEY).map(Boolean::valueOf).orElse(false);
diff --git a/tests/plugins/fake-governance-plugin/src/main/java/ce/ComponentBombReportAnalysisComponentProvider.java b/tests/plugins/fake-governance-plugin/src/main/java/ce/ComponentBombReportAnalysisComponentProvider.java
index a5fe3ee89c4..166f526fca4 100644
--- a/tests/plugins/fake-governance-plugin/src/main/java/ce/ComponentBombReportAnalysisComponentProvider.java
+++ b/tests/plugins/fake-governance-plugin/src/main/java/ce/ComponentBombReportAnalysisComponentProvider.java
@@ -36,6 +36,12 @@ public class ComponentBombReportAnalysisComponentProvider implements ReportAnaly
 
   @Override
   public List<Object> getComponents() {
+    if (bombConfig.isOomStartBomb()) {
+      return singletonList(OOMFailingStartComponent.class);
+    }
+    if (bombConfig.isIseStartBomb()) {
+      return singletonList(ISEFailingStartComponent.class);
+    }
     if (bombConfig.isOomStopBomb()) {
       return singletonList(OOMFailingStopComponent.class);
     }
@@ -46,6 +52,34 @@ public class ComponentBombReportAnalysisComponentProvider implements ReportAnaly
   }
 
   @EagerStart
+  public static final class OOMFailingStartComponent implements Startable {
+
+    @Override
+    public void start() {
+      OOMGenerator.consumeAvailableMemory();
+    }
+
+    @Override
+    public void stop() {
+      // nothing to do
+    }
+  }
+
+  @EagerStart
+  public static final class ISEFailingStartComponent implements Startable {
+
+    @Override
+    public void start() {
+      throw new IllegalStateException("Faking an IllegalStateException thrown by a startable component in the Analysis Report processing container");
+    }
+
+    @Override
+    public void stop() {
+      // nothing to do
+    }
+  }
+
+  @EagerStart
   public static final class OOMFailingStopComponent implements Startable {
 
     @Override
diff --git a/tests/plugins/fake-governance-plugin/src/main/java/ce/ws/BombActivatorAction.java b/tests/plugins/fake-governance-plugin/src/main/java/ce/ws/BombActivatorAction.java
index 2ab01924f0a..e2366b8e16a 100644
--- a/tests/plugins/fake-governance-plugin/src/main/java/ce/ws/BombActivatorAction.java
+++ b/tests/plugins/fake-governance-plugin/src/main/java/ce/ws/BombActivatorAction.java
@@ -20,10 +20,13 @@
 package ce.ws;
 
 import ce.BombConfig;
+import java.util.Arrays;
 import org.sonar.api.server.ws.Request;
 import org.sonar.api.server.ws.Response;
 import org.sonar.api.server.ws.WebService;
 
+import static java.util.stream.Collectors.toList;
+
 public class BombActivatorAction implements FakeGoVWsAction {
 
   private static final String PARAM_BOMB_TYPE = "type";
@@ -41,20 +44,25 @@ public class BombActivatorAction implements FakeGoVWsAction {
       .setHandler(this);
     action.createParam(PARAM_BOMB_TYPE)
       .setRequired(true)
-      .setPossibleValues("OOM", "ISE", "NONE");
+      .setPossibleValues(Arrays.stream(BombType.values()).map(Enum::toString).collect(toList()));
   }
 
   @Override
   public void handle(Request request, Response response) throws Exception {
     BombType bombType = BombType.valueOf(request.mandatoryParam(PARAM_BOMB_TYPE));
 
-    bombConfig.setIseStopBomb(false);
-    bombConfig.setOomStopBomb(false);
+    bombConfig.reset();
     switch (bombType) {
-      case ISE:
+      case ISE_START:
+        bombConfig.setIseStartBomb(true);
+        break;
+      case OOM_START:
+        bombConfig.setOomStartBomb(true);
+        break;
+      case ISE_STOP:
         bombConfig.setIseStopBomb(true);
         break;
-      case OOM:
+      case OOM_STOP:
         bombConfig.setOomStopBomb(true);
         break;
       case NONE:
@@ -67,7 +75,7 @@ public class BombActivatorAction implements FakeGoVWsAction {
   }
 
   enum BombType {
-    NONE, OOM, ISE
+    NONE, OOM_START, ISE_START, OOM_STOP, ISE_STOP
 
   }
 
diff --git a/tests/plugins/fake-governance-plugin/src/main/java/ce/ws/SubmitAction.java b/tests/plugins/fake-governance-plugin/src/main/java/ce/ws/SubmitAction.java
index 9eb607ab789..42f037f819c 100644
--- a/tests/plugins/fake-governance-plugin/src/main/java/ce/ws/SubmitAction.java
+++ b/tests/plugins/fake-governance-plugin/src/main/java/ce/ws/SubmitAction.java
@@ -23,7 +23,6 @@ import org.sonar.api.server.ws.Request;
 import org.sonar.api.server.ws.Response;
 import org.sonar.api.server.ws.WebService;
 import org.sonar.ce.queue.CeQueue;
-import org.sonar.ce.queue.CeTask;
 import org.sonar.ce.queue.CeTaskSubmit;
 
 public class SubmitAction implements FakeGoVWsAction {
@@ -53,7 +52,7 @@ public class SubmitAction implements FakeGoVWsAction {
     CeTaskSubmit.Builder submit = ceQueue.prepareSubmit();
     submit.setType(type);
 
-    CeTask ceTask = ceQueue.submit(submit.build());
+    ceQueue.submit(submit.build());
     response.noContent();
   }
 }
diff --git a/tests/src/test/java/org/sonarqube/tests/ce/CeWorkersTest.java b/tests/src/test/java/org/sonarqube/tests/ce/CeWorkersTest.java
index dc555b5a719..64d0625e192 100644
--- a/tests/src/test/java/org/sonarqube/tests/ce/CeWorkersTest.java
+++ b/tests/src/test/java/org/sonarqube/tests/ce/CeWorkersTest.java
@@ -38,7 +38,9 @@ import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import javax.annotation.Nullable;
+import org.junit.After;
 import org.junit.AfterClass;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
 import org.junit.Test;
@@ -99,6 +101,28 @@ public class CeWorkersTest {
     }
   }
 
+  @Before
+  public void setup() throws Exception {
+    unlockWorkersAndResetWorkerCount();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    unlockWorkersAndResetWorkerCount();
+  }
+
+  private void unlockWorkersAndResetWorkerCount() throws IOException {
+    RandomAccessFile randomAccessFile = null;
+    try {
+      randomAccessFile = new RandomAccessFile(sharedMemory, "rw");
+      MappedByteBuffer mappedByteBuffer = initMappedByteBuffer(randomAccessFile);
+      releaseAnyAnalysisWithFakeGovernancePlugin(mappedByteBuffer);
+      updateWorkerCount(1);
+    } finally {
+      close(randomAccessFile);
+    }
+  }
+
   @Test
   public void ce_worker_is_resilient_to_OOM_and_ISE_during_processing_of_a_task() throws InterruptedException {
     submitFakeTask("OOM");
@@ -149,41 +173,65 @@ public class CeWorkersTest {
   }
 
   @Test
-  public void ce_worker_is_resilient_to_OOM_and_RuntimeException_when_stopping_analysis_report_container() throws IOException {
+  public void ce_worker_is_resilient_to_OOM_and_RuntimeException_when_starting_or_stopping_analysis_report_container() throws IOException {
+    int initSuccessReportTaskCount = adminWsClient.ce().activity(new ActivityWsRequest()
+      .setType("REPORT")
+      .setStatus(ImmutableList.of("SUCCESS")))
+      .getTasksCount();
+    int initFailedReportTaskCount = adminWsClient.ce().activity(new ActivityWsRequest()
+      .setType("REPORT")
+      .setStatus(ImmutableList.of("FAILED")))
+      .getTasksCount();
 
-    RandomAccessFile randomAccessFile = null;
-    try {
-      randomAccessFile = new RandomAccessFile(sharedMemory, "rw");
-      MappedByteBuffer mappedByteBuffer = initMappedByteBuffer(randomAccessFile);
-      releaseAnyAnalysisWithFakeGovernancePlugin(mappedByteBuffer);
+    SonarScanner sonarRunner = SonarScanner.create(ItUtils.projectDir("shared/xoo-sample"));
+    orchestrator.executeBuild(sonarRunner, true);
 
-      SonarScanner sonarRunner = SonarScanner.create(ItUtils.projectDir("shared/xoo-sample"));
-      orchestrator.executeBuild(sonarRunner, true);
+    enableComponentBomb("OOM_STOP");
 
-      enableComponentBomb("OOM");
+    orchestrator.executeBuild(sonarRunner, true);
 
-      orchestrator.executeBuild(sonarRunner, true);
+    enableComponentBomb("NONE");
 
-      enableComponentBomb("NONE");
+    orchestrator.executeBuild(sonarRunner, true);
 
-      orchestrator.executeBuild(sonarRunner, true);
+    enableComponentBomb("ISE_START");
 
-      enableComponentBomb("ISE");
+    orchestrator.executeBuild(sonarRunner, true);
 
-      orchestrator.executeBuild(sonarRunner, true);
+    enableComponentBomb("NONE");
 
-      enableComponentBomb("NONE");
+    orchestrator.executeBuild(sonarRunner, true);
 
-      orchestrator.executeBuild(sonarRunner, true);
+    enableComponentBomb("ISE_STOP");
+
+    orchestrator.executeBuild(sonarRunner, true);
+
+    enableComponentBomb("NONE");
+
+    orchestrator.executeBuild(sonarRunner, true);
+
+    enableComponentBomb("OOM_START");
+
+    orchestrator.executeBuild(sonarRunner, true);
+
+    enableComponentBomb("NONE");
+
+    orchestrator.executeBuild(sonarRunner, true);
+
+    // failure while starting components does fail the tasks
+    assertThat(adminWsClient.ce().activity(new ActivityWsRequest()
+      .setType("REPORT")
+      .setStatus(ImmutableList.of("FAILED")))
+      .getTasksCount())
+        .isEqualTo(initFailedReportTaskCount + 2);
+
+    // failure while stopping components does not fail the tasks
+    assertThat(adminWsClient.ce().activity(new ActivityWsRequest()
+      .setType("REPORT")
+      .setStatus(ImmutableList.of("SUCCESS")))
+      .getTasksCount())
+        .isEqualTo(initSuccessReportTaskCount + 7);
 
-      assertThat(adminWsClient.ce().activity(new ActivityWsRequest()
-        .setType("REPORT")
-        .setStatus(ImmutableList.of("SUCCESS")))
-        .getTasksCount())
-          .isEqualTo(5);
-    } finally {
-      close(randomAccessFile);
-    }
   }
 
   private void enableComponentBomb(String type) {
@@ -332,12 +380,17 @@ public class CeWorkersTest {
   }
 
   private void waitForEmptyQueue() throws InterruptedException {
+    int delay = 200;
+    int timeout = 5 * 10; // 10 seconds
+    int i = 0;
     int tasksCount;
     do {
-      Thread.sleep(100);
+      Thread.sleep(delay);
       tasksCount = adminWsClient.ce().activity(new ActivityWsRequest()
         .setStatus(ImmutableList.of("PENDING", "IN_PROGRESS")))
         .getTasksCount();
-    } while (tasksCount > 0);
+      i++;
+    } while (i <= timeout && tasksCount > 0);
+    assertThat(tasksCount).describedAs("Failed to get to an empty CE queue in a timely fashion").isZero();
   }
 }