diff options
author | Duarte Meneses <duarte.meneses@sonarsource.com> | 2018-06-20 10:05:38 +0200 |
---|---|---|
committer | SonarTech <sonartech@sonarsource.com> | 2018-07-13 20:21:28 +0200 |
commit | d98c3ab5b4bc3615b10ca984681b809e98556450 (patch) | |
tree | b5b29f35f898cc044fd13a7189504ee3fc0e578c /server/sonar-ce | |
parent | 0c4bdce520ccca20bd5e1b92a7c5d05821b53960 (diff) | |
download | sonarqube-d98c3ab5b4bc3615b10ca984681b809e98556450.tar.gz sonarqube-d98c3ab5b4bc3615b10ca984681b809e98556450.zip |
SONAR-10898 Number of CE workers not taken into account in all nodes
Diffstat (limited to 'server/sonar-ce')
12 files changed, 28 insertions, 263 deletions
diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/CeHttpModule.java b/server/sonar-ce/src/main/java/org/sonar/ce/CeHttpModule.java index 08e5af8ecfe..223e289811b 100644 --- a/server/sonar-ce/src/main/java/org/sonar/ce/CeHttpModule.java +++ b/server/sonar-ce/src/main/java/org/sonar/ce/CeHttpModule.java @@ -22,7 +22,6 @@ package org.sonar.ce; import org.sonar.ce.httpd.CeHttpServer; import org.sonar.ce.logging.ChangeLogLevelHttpAction; import org.sonar.ce.systeminfo.SystemInfoHttpAction; -import org.sonar.ce.taskprocessor.RefreshWorkerCountAction; import org.sonar.core.platform.Module; public class CeHttpModule extends Module { @@ -31,7 +30,6 @@ public class CeHttpModule extends Module { add( CeHttpServer.class, SystemInfoHttpAction.class, - ChangeLogLevelHttpAction.class, - RefreshWorkerCountAction.class); + ChangeLogLevelHttpAction.class); } } diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/configuration/CeConfiguration.java b/server/sonar-ce/src/main/java/org/sonar/ce/configuration/CeConfiguration.java index 8b040467e4b..a44cbd0c650 100644 --- a/server/sonar-ce/src/main/java/org/sonar/ce/configuration/CeConfiguration.java +++ b/server/sonar-ce/src/main/java/org/sonar/ce/configuration/CeConfiguration.java @@ -22,11 +22,6 @@ package org.sonar.ce.configuration; public interface CeConfiguration { /** - * Requests {@link CeConfiguration} to refresh its state, if it has any. - */ - void refresh(); - - /** * The maximum number of workers to process CeTasks concurrently, integer strictly greater than 0. */ int getWorkerMaxCount(); diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/configuration/CeConfigurationImpl.java b/server/sonar-ce/src/main/java/org/sonar/ce/configuration/CeConfigurationImpl.java index 53fd4707614..b2956f44f11 100644 --- a/server/sonar-ce/src/main/java/org/sonar/ce/configuration/CeConfigurationImpl.java +++ b/server/sonar-ce/src/main/java/org/sonar/ce/configuration/CeConfigurationImpl.java @@ -20,6 +20,7 @@ package org.sonar.ce.configuration; import javax.annotation.CheckForNull; +import javax.annotation.Nullable; import org.picocontainer.Startable; import org.sonar.api.config.Configuration; import org.sonar.api.utils.MessageException; @@ -53,20 +54,22 @@ public class CeConfigurationImpl implements CeConfiguration, Startable { private int workerCount; public CeConfigurationImpl(Configuration configuration) { - this.workerCountProvider = null; - this.workerThreadCount = DEFAULT_WORKER_THREAD_COUNT; - this.workerCount = DEFAULT_WORKER_COUNT; - this.gracefultStopTimeoutInMs = configuration.getInt(SONAR_CE_GRACEFUL_STOP_TIME_OUT_IN_MS).orElse(GRACEFUL_STOP_TIMEOUT); + this(configuration, null); } - public CeConfigurationImpl(Configuration configuration, WorkerCountProvider workerCountProvider) { + public CeConfigurationImpl(Configuration configuration, @Nullable WorkerCountProvider workerCountProvider) { this.workerCountProvider = workerCountProvider; - this.workerThreadCount = MAX_WORKER_THREAD_COUNT; - this.workerCount = readWorkerCount(workerCountProvider); this.gracefultStopTimeoutInMs = configuration.getInt(SONAR_CE_GRACEFUL_STOP_TIME_OUT_IN_MS).orElse(GRACEFUL_STOP_TIMEOUT); + if (workerCountProvider == null) { + this.workerCount = DEFAULT_WORKER_COUNT; + this.workerThreadCount = DEFAULT_WORKER_THREAD_COUNT; + } else { + this.workerCount = readWorkerCount(workerCountProvider); + this.workerThreadCount = MAX_WORKER_THREAD_COUNT; + } } - private static int readWorkerCount(WorkerCountProvider workerCountProvider) { + private static synchronized int readWorkerCount(WorkerCountProvider workerCountProvider) { int value = workerCountProvider.get(); if (value < DEFAULT_WORKER_COUNT || value > MAX_WORKER_THREAD_COUNT) { throw parsingError(value); @@ -76,13 +79,13 @@ public class CeConfigurationImpl implements CeConfiguration, Startable { private static MessageException parsingError(int value) { return MessageException.of(format( - "Worker count '%s' is invalid. It must an integer strictly greater than 0 and less or equal to 10", - value)); + "Worker count '%s' is invalid. It must be an integer strictly greater than 0 and less or equal to 10", + value)); } @Override public void start() { - // + // nothing to do } @Override @@ -91,19 +94,15 @@ public class CeConfigurationImpl implements CeConfiguration, Startable { } @Override - public void refresh() { - if (workerCountProvider != null) { - this.workerCount = readWorkerCount(workerCountProvider); - } - } - - @Override public int getWorkerMaxCount() { return workerThreadCount; } @Override public int getWorkerCount() { + if (workerCountProvider != null) { + workerCount = readWorkerCount(workerCountProvider); + } return workerCount; } diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/taskprocessor/EnabledCeWorkerController.java b/server/sonar-ce/src/main/java/org/sonar/ce/taskprocessor/EnabledCeWorkerController.java index 7757f08ac59..e11063d311c 100644 --- a/server/sonar-ce/src/main/java/org/sonar/ce/taskprocessor/EnabledCeWorkerController.java +++ b/server/sonar-ce/src/main/java/org/sonar/ce/taskprocessor/EnabledCeWorkerController.java @@ -28,11 +28,6 @@ public interface EnabledCeWorkerController { } /** - * Requests the {@link EnabledCeWorkerController} to refresh its state, if it has any. - */ - void refresh(); - - /** * Returns {@code true} if the specified {@link CeWorker} is enabled */ boolean isEnabled(CeWorker ceWorker); diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/taskprocessor/EnabledCeWorkerControllerImpl.java b/server/sonar-ce/src/main/java/org/sonar/ce/taskprocessor/EnabledCeWorkerControllerImpl.java index a438ad8bd69..7370647a589 100644 --- a/server/sonar-ce/src/main/java/org/sonar/ce/taskprocessor/EnabledCeWorkerControllerImpl.java +++ b/server/sonar-ce/src/main/java/org/sonar/ce/taskprocessor/EnabledCeWorkerControllerImpl.java @@ -20,14 +20,12 @@ package org.sonar.ce.taskprocessor; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; import org.sonar.api.utils.log.Loggers; import org.sonar.ce.configuration.CeConfiguration; public class EnabledCeWorkerControllerImpl implements EnabledCeWorkerController { private final ConcurrentHashMap<CeWorker, Status> map = new ConcurrentHashMap<>(); private final CeConfiguration ceConfiguration; - private final AtomicInteger workerCount; enum Status { PROCESSING, PAUSED @@ -35,24 +33,17 @@ public class EnabledCeWorkerControllerImpl implements EnabledCeWorkerController public EnabledCeWorkerControllerImpl(CeConfiguration ceConfiguration) { this.ceConfiguration = ceConfiguration; - this.workerCount = new AtomicInteger(ceConfiguration.getWorkerCount()); logEnabledWorkerCount(); } private void logEnabledWorkerCount() { - if (workerCount.get() > 1) { - Loggers.get(EnabledCeWorkerController.class).info("Compute Engine will use {} concurrent workers to process tasks", this.workerCount); + int workerCount = ceConfiguration.getWorkerCount(); + if (workerCount > 1) { + Loggers.get(EnabledCeWorkerController.class).info("Compute Engine will use {} concurrent workers to process tasks", workerCount); } } @Override - public void refresh() { - ceConfiguration.refresh(); - this.workerCount.set(ceConfiguration.getWorkerCount()); - logEnabledWorkerCount(); - } - - @Override public ProcessingRecorderHook registerProcessingFor(CeWorker ceWorker) { return new ProcessingRecorderHookImpl(ceWorker); } @@ -70,7 +61,7 @@ public class EnabledCeWorkerControllerImpl implements EnabledCeWorkerController */ @Override public boolean isEnabled(CeWorker ceWorker) { - return ceWorker.getOrdinal() < workerCount.get(); + return ceWorker.getOrdinal() < ceConfiguration.getWorkerCount(); } private class ProcessingRecorderHookImpl implements ProcessingRecorderHook { diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/taskprocessor/RefreshWorkerCountAction.java b/server/sonar-ce/src/main/java/org/sonar/ce/taskprocessor/RefreshWorkerCountAction.java deleted file mode 100644 index 225f457be84..00000000000 --- a/server/sonar-ce/src/main/java/org/sonar/ce/taskprocessor/RefreshWorkerCountAction.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2018 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.ce.taskprocessor; - -import fi.iki.elonen.NanoHTTPD; -import org.sonar.ce.httpd.HttpAction; - -import static fi.iki.elonen.NanoHTTPD.MIME_PLAINTEXT; -import static fi.iki.elonen.NanoHTTPD.newFixedLengthResponse; -import static fi.iki.elonen.NanoHTTPD.Response.Status.METHOD_NOT_ALLOWED; -import static fi.iki.elonen.NanoHTTPD.Response.Status.OK; - -public class RefreshWorkerCountAction implements HttpAction { - private static final String PATH = "refreshWorkerCount"; - - private final EnabledCeWorkerController enabledCeWorkerController; - - public RefreshWorkerCountAction(EnabledCeWorkerController enabledCeWorkerController) { - this.enabledCeWorkerController = enabledCeWorkerController; - } - - @Override - public void register(ActionRegistry registry) { - registry.register(PATH, this); - } - - @Override - public NanoHTTPD.Response serve(NanoHTTPD.IHTTPSession session) { - if (session.getMethod() != NanoHTTPD.Method.POST) { - return newFixedLengthResponse(METHOD_NOT_ALLOWED, MIME_PLAINTEXT, null); - } - - enabledCeWorkerController.refresh(); - - return newFixedLengthResponse(OK, MIME_PLAINTEXT, null); - } -} diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/configuration/CeConfigurationImplTest.java b/server/sonar-ce/src/test/java/org/sonar/ce/configuration/CeConfigurationImplTest.java index 29068c986a8..5eb33c75c05 100644 --- a/server/sonar-ce/src/test/java/org/sonar/ce/configuration/CeConfigurationImplTest.java +++ b/server/sonar-ce/src/test/java/org/sonar/ce/configuration/CeConfigurationImplTest.java @@ -20,7 +20,6 @@ package org.sonar.ce.configuration; import java.util.Random; -import java.util.stream.IntStream; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -96,7 +95,7 @@ public class CeConfigurationImplTest { private void expectMessageException(int value) { expectedException.expect(MessageException.class); expectedException.expectMessage("Worker count '" + value + "' is invalid. " + - "It must an integer strictly greater than 0 and less or equal to 10"); + "It must be an integer strictly greater than 0 and less or equal to 10"); } @Test @@ -117,50 +116,6 @@ public class CeConfigurationImplTest { .isEqualTo(2L); } - @Test - public void refresh_does_not_change_any_value_when_there_is_no_WorkerCountProvider() { - CeConfigurationImpl underTest = new CeConfigurationImpl(EMPTY_CONFIGURATION); - long cleanCeTasksInitialDelay = underTest.getCleanCeTasksInitialDelay(); - long cleanCeTasksDelay = underTest.getCleanCeTasksDelay(); - long queuePollingDelay = underTest.getQueuePollingDelay(); - int workerThreadCount = underTest.getWorkerMaxCount(); - int workerCount = underTest.getWorkerCount(); - - IntStream.range(0, 1 + abs(new Random().nextInt(10))) - .forEach(ignored -> { - underTest.refresh(); - assertThat(underTest.getCleanCeTasksInitialDelay()).isEqualTo(cleanCeTasksInitialDelay); - assertThat(underTest.getCleanCeTasksDelay()).isEqualTo(cleanCeTasksDelay); - assertThat(underTest.getQueuePollingDelay()).isEqualTo(queuePollingDelay); - assertThat(underTest.getWorkerMaxCount()).isEqualTo(workerThreadCount); - assertThat(underTest.getWorkerCount()).isEqualTo(workerCount); - }); - } - - @Test - public void refresh_updates_only_workerCount_from_WorkerCountProvider_when_there_WorkerCountProvider_is_present() { - workerCountProvider.set(randomValidWorkerCount()); - CeConfigurationImpl underTest = new CeConfigurationImpl(EMPTY_CONFIGURATION, workerCountProvider); - long cleanCeTasksInitialDelay = underTest.getCleanCeTasksInitialDelay(); - long cleanCeTasksDelay = underTest.getCleanCeTasksDelay(); - long queuePollingDelay = underTest.getQueuePollingDelay(); - int workerThreadCount = underTest.getWorkerMaxCount(); - - IntStream.range(0, 1 + abs(new Random().nextInt(10))) - .forEach(ignored -> { - int newWorkerCount = randomValidWorkerCount(); - workerCountProvider.set(newWorkerCount); - - underTest.refresh(); - - assertThat(underTest.getCleanCeTasksInitialDelay()).isEqualTo(cleanCeTasksInitialDelay); - assertThat(underTest.getCleanCeTasksDelay()).isEqualTo(cleanCeTasksDelay); - assertThat(underTest.getQueuePollingDelay()).isEqualTo(queuePollingDelay); - assertThat(underTest.getWorkerMaxCount()).isEqualTo(workerThreadCount); - assertThat(underTest.getWorkerCount()).isEqualTo(newWorkerCount); - }); - } - private static final class SimpleWorkerCountProvider implements WorkerCountProvider { private int value = 0; diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/configuration/CeConfigurationRule.java b/server/sonar-ce/src/test/java/org/sonar/ce/configuration/CeConfigurationRule.java index 86d9d87a49f..6468e34a314 100644 --- a/server/sonar-ce/src/test/java/org/sonar/ce/configuration/CeConfigurationRule.java +++ b/server/sonar-ce/src/test/java/org/sonar/ce/configuration/CeConfigurationRule.java @@ -19,7 +19,6 @@ */ package org.sonar.ce.configuration; -import java.util.function.Consumer; import org.junit.rules.ExternalResource; import static com.google.common.base.Preconditions.checkArgument; @@ -33,18 +32,6 @@ public class CeConfigurationRule extends ExternalResource implements CeConfigura private long queuePollingDelay = 2 * 1000L; private long cancelWornOutsInitialDelay = 1L; private long cancelWornOutsDelay = 10L; - private Consumer<CeConfigurationRule> refreshCallHook; - - @Override - public void refresh() { - if (this.refreshCallHook != null) { - this.refreshCallHook.accept(this); - } - } - - public void setRefreshCallHook(Consumer<CeConfigurationRule> refreshCallHook) { - this.refreshCallHook = refreshCallHook; - } @Override public int getWorkerMaxCount() { diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java b/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java index c9efd801a37..0ed29601675 100644 --- a/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java +++ b/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java @@ -100,7 +100,7 @@ public class ComputeEngineContainerImplTest { + 72 // level 4 + 6 // content of CeConfigurationModule + 4 // content of CeQueueModule - + 4 // content of CeHttpModule + + 3 // content of CeHttpModule + 3 // content of CeTaskCommonsModule + 4 // content of ProjectAnalysisTaskModule + 7 // content of CeTaskProcessorModule diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/monitoring/CeTasksMBeanImplTest.java b/server/sonar-ce/src/test/java/org/sonar/ce/monitoring/CeTasksMBeanImplTest.java index 33e3416e8a6..cada6ab59aa 100644 --- a/server/sonar-ce/src/test/java/org/sonar/ce/monitoring/CeTasksMBeanImplTest.java +++ b/server/sonar-ce/src/test/java/org/sonar/ce/monitoring/CeTasksMBeanImplTest.java @@ -145,11 +145,6 @@ public class CeTasksMBeanImplTest { private static class DumbCeConfiguration implements CeConfiguration { @Override - public void refresh() { - throw new UnsupportedOperationException("Refresh is not implemented"); - } - - @Override public int getWorkerMaxCount() { return WORKER_MAX_COUNT; } diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/taskprocessor/EnabledCeWorkerControllerImplTest.java b/server/sonar-ce/src/test/java/org/sonar/ce/taskprocessor/EnabledCeWorkerControllerImplTest.java index c2c7726d2be..54c90fb1b7f 100644 --- a/server/sonar-ce/src/test/java/org/sonar/ce/taskprocessor/EnabledCeWorkerControllerImplTest.java +++ b/server/sonar-ce/src/test/java/org/sonar/ce/taskprocessor/EnabledCeWorkerControllerImplTest.java @@ -93,46 +93,16 @@ public class EnabledCeWorkerControllerImplTest { } @Test - public void workerCount_is_loaded_in_constructor() { - when(ceWorker.getOrdinal()).thenReturn(randomWorkerCount); - assertThat(underTest.isEnabled(ceWorker)).isFalse(); - - ceConfigurationRule.setWorkerCount(randomWorkerCount + 1); - assertThat(underTest.isEnabled(ceWorker)).isFalse(); - } + public void workerCount_is_always_reloaded() { + when(ceWorker.getOrdinal()).thenReturn(1); - @Test - public void refresh_reloads_workerCount() { - when(ceWorker.getOrdinal()).thenReturn(randomWorkerCount); + ceConfigurationRule.setWorkerCount(1); assertThat(underTest.isEnabled(ceWorker)).isFalse(); - ceConfigurationRule.setRefreshCallHook((rule) -> rule.setWorkerCount(randomWorkerCount + 1)); - - underTest.refresh(); + ceConfigurationRule.setWorkerCount(2); assertThat(underTest.isEnabled(ceWorker)).isTrue(); } - @Test - public void refresh_writes_info_log_if_workerCount_is_greater_than_1() { - logTester.clear(); - int newWorkerCount = randomWorkerCount + 1; - ceConfigurationRule.setRefreshCallHook((rule) -> rule.setWorkerCount(newWorkerCount)); - - underTest.refresh(); - - verifyInfoLog(newWorkerCount); - } - - @Test - public void refresh_writes_no_info_log_if_workerCount_is_1() { - logTester.clear(); - ceConfigurationRule.setRefreshCallHook((rule) -> rule.setWorkerCount(1)); - - underTest.refresh(); - - assertThat(logTester.logs()).isEmpty(); - } - private void verifyInfoLog(int workerCount) { assertThat(logTester.logs()).hasSize(1); assertThat(logTester.logs(LoggerLevel.INFO)) diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/taskprocessor/RefreshWorkerCountActionTest.java b/server/sonar-ce/src/test/java/org/sonar/ce/taskprocessor/RefreshWorkerCountActionTest.java deleted file mode 100644 index ea9a3a7037d..00000000000 --- a/server/sonar-ce/src/test/java/org/sonar/ce/taskprocessor/RefreshWorkerCountActionTest.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2018 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.ce.taskprocessor; - -import fi.iki.elonen.NanoHTTPD; -import org.junit.Test; -import org.sonar.ce.httpd.HttpAction; - -import static fi.iki.elonen.NanoHTTPD.Method.GET; -import static fi.iki.elonen.NanoHTTPD.Method.POST; -import static fi.iki.elonen.NanoHTTPD.Response.Status.METHOD_NOT_ALLOWED; -import static fi.iki.elonen.NanoHTTPD.Response.Status.OK; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.verifyZeroInteractions; -import static org.sonar.ce.httpd.CeHttpUtils.createHttpSession; - -public class RefreshWorkerCountActionTest { - private EnabledCeWorkerController enabledCeWorkerController = mock(EnabledCeWorkerController.class); - private RefreshWorkerCountAction underTest = new RefreshWorkerCountAction(enabledCeWorkerController); - - @Test - public void register_to_path_changeLogLevel() { - HttpAction.ActionRegistry actionRegistry = mock(HttpAction.ActionRegistry.class); - - underTest.register(actionRegistry); - - verify(actionRegistry).register("refreshWorkerCount", underTest); - } - - @Test - public void serves_METHOD_NOT_ALLOWED_error_when_method_is_not_POST() { - NanoHTTPD.Response response = underTest.serve(createHttpSession(GET)); - - assertThat(response.getStatus()).isEqualTo(METHOD_NOT_ALLOWED); - verifyZeroInteractions(enabledCeWorkerController); - } - - @Test - public void call_EnabledCeWorkerController_refresh_on_POST() { - NanoHTTPD.Response response = underTest.serve(createHttpSession(POST)); - - assertThat(response.getStatus()).isEqualTo(OK); - verify(enabledCeWorkerController).refresh(); - verifyNoMoreInteractions(enabledCeWorkerController); - } -} |