From: Sébastien Lesaint Date: Thu, 2 Nov 2017 10:47:13 +0000 (+0100) Subject: SONAR-9973 IT for report processing resilient to failure during start X-Git-Tag: 6.7~7 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=refs%2Fpull%2F2780%2Fhead;p=sonarqube.git SONAR-9973 IT for report processing resilient to failure during start of components --- diff --git a/tests/plugins/fake-governance-plugin/src/main/java/ce/BombConfig.java b/tests/plugins/fake-governance-plugin/src/main/java/ce/BombConfig.java index 52207065d1a..3e5af086b86 100644 --- a/tests/plugins/fake-governance-plugin/src/main/java/ce/BombConfig.java +++ b/tests/plugins/fake-governance-plugin/src/main/java/ce/BombConfig.java @@ -27,6 +27,8 @@ import org.sonar.db.DbSession; @ServerSide @ComputeEngineSide public class BombConfig { + private static final String OOM_START_BOMB_KEY = "oomStartBomb"; + private static final String ISE_START_BOMB_KEY = "iseStartBomb"; private static final String OOM_STOP_BOMB_KEY = "oomStopBomb"; private static final String ISE_STOP_BOMB_KEY = "iseStopBomb"; @@ -36,6 +38,42 @@ public class BombConfig { this.dbClient = dbClient; } + public void reset() { + try (DbSession dbSession = dbClient.openSession(false)) { + dbClient.internalPropertiesDao().save(dbSession, OOM_START_BOMB_KEY, String.valueOf(false)); + dbClient.internalPropertiesDao().save(dbSession, ISE_START_BOMB_KEY, String.valueOf(false)); + dbClient.internalPropertiesDao().save(dbSession, OOM_STOP_BOMB_KEY, String.valueOf(false)); + dbClient.internalPropertiesDao().save(dbSession, ISE_STOP_BOMB_KEY, String.valueOf(false)); + dbSession.commit(); + } + } + + public boolean isOomStartBomb() { + try (DbSession dbSession = dbClient.openSession(false)) { + return dbClient.internalPropertiesDao().selectByKey(dbSession, OOM_START_BOMB_KEY).map(Boolean::valueOf).orElse(false); + } + } + + public void setOomStartBomb(boolean oomStartBomb) { + try (DbSession dbSession = dbClient.openSession(false)) { + dbClient.internalPropertiesDao().save(dbSession, OOM_START_BOMB_KEY, String.valueOf(oomStartBomb)); + dbSession.commit(); + } + } + + public boolean isIseStartBomb() { + try (DbSession dbSession = dbClient.openSession(false)) { + return dbClient.internalPropertiesDao().selectByKey(dbSession, ISE_START_BOMB_KEY).map(Boolean::valueOf).orElse(false); + } + } + + public void setIseStartBomb(boolean iseStartBomb) { + try (DbSession dbSession = dbClient.openSession(false)) { + dbClient.internalPropertiesDao().save(dbSession, ISE_START_BOMB_KEY, String.valueOf(iseStartBomb)); + dbSession.commit(); + } + } + public boolean isOomStopBomb() { try (DbSession dbSession = dbClient.openSession(false)) { return dbClient.internalPropertiesDao().selectByKey(dbSession, OOM_STOP_BOMB_KEY).map(Boolean::valueOf).orElse(false); diff --git a/tests/plugins/fake-governance-plugin/src/main/java/ce/ComponentBombReportAnalysisComponentProvider.java b/tests/plugins/fake-governance-plugin/src/main/java/ce/ComponentBombReportAnalysisComponentProvider.java index a5fe3ee89c4..166f526fca4 100644 --- a/tests/plugins/fake-governance-plugin/src/main/java/ce/ComponentBombReportAnalysisComponentProvider.java +++ b/tests/plugins/fake-governance-plugin/src/main/java/ce/ComponentBombReportAnalysisComponentProvider.java @@ -36,6 +36,12 @@ public class ComponentBombReportAnalysisComponentProvider implements ReportAnaly @Override public List getComponents() { + if (bombConfig.isOomStartBomb()) { + return singletonList(OOMFailingStartComponent.class); + } + if (bombConfig.isIseStartBomb()) { + return singletonList(ISEFailingStartComponent.class); + } if (bombConfig.isOomStopBomb()) { return singletonList(OOMFailingStopComponent.class); } @@ -45,6 +51,34 @@ public class ComponentBombReportAnalysisComponentProvider implements ReportAnaly return emptyList(); } + @EagerStart + public static final class OOMFailingStartComponent implements Startable { + + @Override + public void start() { + OOMGenerator.consumeAvailableMemory(); + } + + @Override + public void stop() { + // nothing to do + } + } + + @EagerStart + public static final class ISEFailingStartComponent implements Startable { + + @Override + public void start() { + throw new IllegalStateException("Faking an IllegalStateException thrown by a startable component in the Analysis Report processing container"); + } + + @Override + public void stop() { + // nothing to do + } + } + @EagerStart public static final class OOMFailingStopComponent implements Startable { diff --git a/tests/plugins/fake-governance-plugin/src/main/java/ce/ws/BombActivatorAction.java b/tests/plugins/fake-governance-plugin/src/main/java/ce/ws/BombActivatorAction.java index 2ab01924f0a..e2366b8e16a 100644 --- a/tests/plugins/fake-governance-plugin/src/main/java/ce/ws/BombActivatorAction.java +++ b/tests/plugins/fake-governance-plugin/src/main/java/ce/ws/BombActivatorAction.java @@ -20,10 +20,13 @@ package ce.ws; import ce.BombConfig; +import java.util.Arrays; import org.sonar.api.server.ws.Request; import org.sonar.api.server.ws.Response; import org.sonar.api.server.ws.WebService; +import static java.util.stream.Collectors.toList; + public class BombActivatorAction implements FakeGoVWsAction { private static final String PARAM_BOMB_TYPE = "type"; @@ -41,20 +44,25 @@ public class BombActivatorAction implements FakeGoVWsAction { .setHandler(this); action.createParam(PARAM_BOMB_TYPE) .setRequired(true) - .setPossibleValues("OOM", "ISE", "NONE"); + .setPossibleValues(Arrays.stream(BombType.values()).map(Enum::toString).collect(toList())); } @Override public void handle(Request request, Response response) throws Exception { BombType bombType = BombType.valueOf(request.mandatoryParam(PARAM_BOMB_TYPE)); - bombConfig.setIseStopBomb(false); - bombConfig.setOomStopBomb(false); + bombConfig.reset(); switch (bombType) { - case ISE: + case ISE_START: + bombConfig.setIseStartBomb(true); + break; + case OOM_START: + bombConfig.setOomStartBomb(true); + break; + case ISE_STOP: bombConfig.setIseStopBomb(true); break; - case OOM: + case OOM_STOP: bombConfig.setOomStopBomb(true); break; case NONE: @@ -67,7 +75,7 @@ public class BombActivatorAction implements FakeGoVWsAction { } enum BombType { - NONE, OOM, ISE + NONE, OOM_START, ISE_START, OOM_STOP, ISE_STOP } diff --git a/tests/plugins/fake-governance-plugin/src/main/java/ce/ws/SubmitAction.java b/tests/plugins/fake-governance-plugin/src/main/java/ce/ws/SubmitAction.java index 9eb607ab789..42f037f819c 100644 --- a/tests/plugins/fake-governance-plugin/src/main/java/ce/ws/SubmitAction.java +++ b/tests/plugins/fake-governance-plugin/src/main/java/ce/ws/SubmitAction.java @@ -23,7 +23,6 @@ import org.sonar.api.server.ws.Request; import org.sonar.api.server.ws.Response; import org.sonar.api.server.ws.WebService; import org.sonar.ce.queue.CeQueue; -import org.sonar.ce.queue.CeTask; import org.sonar.ce.queue.CeTaskSubmit; public class SubmitAction implements FakeGoVWsAction { @@ -53,7 +52,7 @@ public class SubmitAction implements FakeGoVWsAction { CeTaskSubmit.Builder submit = ceQueue.prepareSubmit(); submit.setType(type); - CeTask ceTask = ceQueue.submit(submit.build()); + ceQueue.submit(submit.build()); response.noContent(); } } diff --git a/tests/src/test/java/org/sonarqube/tests/ce/CeWorkersTest.java b/tests/src/test/java/org/sonarqube/tests/ce/CeWorkersTest.java index dc555b5a719..64d0625e192 100644 --- a/tests/src/test/java/org/sonarqube/tests/ce/CeWorkersTest.java +++ b/tests/src/test/java/org/sonarqube/tests/ce/CeWorkersTest.java @@ -38,7 +38,9 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.IntStream; import javax.annotation.Nullable; +import org.junit.After; import org.junit.AfterClass; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; @@ -99,6 +101,28 @@ public class CeWorkersTest { } } + @Before + public void setup() throws Exception { + unlockWorkersAndResetWorkerCount(); + } + + @After + public void tearDown() throws Exception { + unlockWorkersAndResetWorkerCount(); + } + + private void unlockWorkersAndResetWorkerCount() throws IOException { + RandomAccessFile randomAccessFile = null; + try { + randomAccessFile = new RandomAccessFile(sharedMemory, "rw"); + MappedByteBuffer mappedByteBuffer = initMappedByteBuffer(randomAccessFile); + releaseAnyAnalysisWithFakeGovernancePlugin(mappedByteBuffer); + updateWorkerCount(1); + } finally { + close(randomAccessFile); + } + } + @Test public void ce_worker_is_resilient_to_OOM_and_ISE_during_processing_of_a_task() throws InterruptedException { submitFakeTask("OOM"); @@ -149,41 +173,65 @@ public class CeWorkersTest { } @Test - public void ce_worker_is_resilient_to_OOM_and_RuntimeException_when_stopping_analysis_report_container() throws IOException { + public void ce_worker_is_resilient_to_OOM_and_RuntimeException_when_starting_or_stopping_analysis_report_container() throws IOException { + int initSuccessReportTaskCount = adminWsClient.ce().activity(new ActivityWsRequest() + .setType("REPORT") + .setStatus(ImmutableList.of("SUCCESS"))) + .getTasksCount(); + int initFailedReportTaskCount = adminWsClient.ce().activity(new ActivityWsRequest() + .setType("REPORT") + .setStatus(ImmutableList.of("FAILED"))) + .getTasksCount(); - RandomAccessFile randomAccessFile = null; - try { - randomAccessFile = new RandomAccessFile(sharedMemory, "rw"); - MappedByteBuffer mappedByteBuffer = initMappedByteBuffer(randomAccessFile); - releaseAnyAnalysisWithFakeGovernancePlugin(mappedByteBuffer); + SonarScanner sonarRunner = SonarScanner.create(ItUtils.projectDir("shared/xoo-sample")); + orchestrator.executeBuild(sonarRunner, true); - SonarScanner sonarRunner = SonarScanner.create(ItUtils.projectDir("shared/xoo-sample")); - orchestrator.executeBuild(sonarRunner, true); + enableComponentBomb("OOM_STOP"); - enableComponentBomb("OOM"); + orchestrator.executeBuild(sonarRunner, true); - orchestrator.executeBuild(sonarRunner, true); + enableComponentBomb("NONE"); - enableComponentBomb("NONE"); + orchestrator.executeBuild(sonarRunner, true); - orchestrator.executeBuild(sonarRunner, true); + enableComponentBomb("ISE_START"); - enableComponentBomb("ISE"); + orchestrator.executeBuild(sonarRunner, true); - orchestrator.executeBuild(sonarRunner, true); + enableComponentBomb("NONE"); - enableComponentBomb("NONE"); + orchestrator.executeBuild(sonarRunner, true); - orchestrator.executeBuild(sonarRunner, true); + enableComponentBomb("ISE_STOP"); + + orchestrator.executeBuild(sonarRunner, true); + + enableComponentBomb("NONE"); + + orchestrator.executeBuild(sonarRunner, true); + + enableComponentBomb("OOM_START"); + + orchestrator.executeBuild(sonarRunner, true); + + enableComponentBomb("NONE"); + + orchestrator.executeBuild(sonarRunner, true); + + // failure while starting components does fail the tasks + assertThat(adminWsClient.ce().activity(new ActivityWsRequest() + .setType("REPORT") + .setStatus(ImmutableList.of("FAILED"))) + .getTasksCount()) + .isEqualTo(initFailedReportTaskCount + 2); + + // failure while stopping components does not fail the tasks + assertThat(adminWsClient.ce().activity(new ActivityWsRequest() + .setType("REPORT") + .setStatus(ImmutableList.of("SUCCESS"))) + .getTasksCount()) + .isEqualTo(initSuccessReportTaskCount + 7); - assertThat(adminWsClient.ce().activity(new ActivityWsRequest() - .setType("REPORT") - .setStatus(ImmutableList.of("SUCCESS"))) - .getTasksCount()) - .isEqualTo(5); - } finally { - close(randomAccessFile); - } } private void enableComponentBomb(String type) { @@ -332,12 +380,17 @@ public class CeWorkersTest { } private void waitForEmptyQueue() throws InterruptedException { + int delay = 200; + int timeout = 5 * 10; // 10 seconds + int i = 0; int tasksCount; do { - Thread.sleep(100); + Thread.sleep(delay); tasksCount = adminWsClient.ce().activity(new ActivityWsRequest() .setStatus(ImmutableList.of("PENDING", "IN_PROGRESS"))) .getTasksCount(); - } while (tasksCount > 0); + i++; + } while (i <= timeout && tasksCount > 0); + assertThat(tasksCount).describedAs("Failed to get to an empty CE queue in a timely fashion").isZero(); } }