SONAR-9802 allow to change the log level of a cluster

19 files changed, 446 insertions, 87 deletions

diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/logging/ChangeLogLevelHttpAction.java b/server/sonar-ce/src/main/java/org/sonar/ce/logging/ChangeLogLevelHttpAction.java
index 3e47ea2a962..aa67614d588 100644
--- a/server/sonar-ce/src/main/java/org/sonar/ce/logging/ChangeLogLevelHttpAction.java
+++ b/server/sonar-ce/src/main/java/org/sonar/ce/logging/ChangeLogLevelHttpAction.java
@@ -23,8 +23,6 @@ import fi.iki.elonen.NanoHTTPD;
 import org.sonar.api.utils.log.LoggerLevel;
 import org.sonar.api.utils.log.Loggers;
 import org.sonar.ce.httpd.HttpAction;
-import org.sonar.ce.log.CeProcessLogging;
-import org.sonar.db.Database;
 import org.sonar.server.platform.ServerLogging;
 
 import static fi.iki.elonen.NanoHTTPD.MIME_PLAINTEXT;
@@ -40,13 +38,9 @@ public class ChangeLogLevelHttpAction implements HttpAction {
   private static final String PARAM_LEVEL = "level";
 
   private final ServerLogging logging;
-  private final Database db;
-  private final CeProcessLogging ceProcessLogging;
 
-  public ChangeLogLevelHttpAction(ServerLogging logging, Database db, CeProcessLogging ceProcessLogging) {
+  public ChangeLogLevelHttpAction(ServerLogging logging) {
     this.logging = logging;
-    this.db = db;
-    this.ceProcessLogging = ceProcessLogging;
   }
 
   @Override
@@ -66,8 +60,7 @@ public class ChangeLogLevelHttpAction implements HttpAction {
     }
     try {
       LoggerLevel level = LoggerLevel.valueOf(levelStr);
-      db.enableSqlLogging(level.equals(LoggerLevel.TRACE));
-      logging.changeLevel(ceProcessLogging, level);
+      logging.changeLevel(level);
       return newFixedLengthResponse(OK, MIME_PLAINTEXT, null);
     } catch (IllegalArgumentException e) {
       Loggers.get(ChangeLogLevelHttpAction.class).debug("Value '{}' for parameter '{}' is invalid", levelStr, PARAM_LEVEL, e);
diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/logging/ChangeLogLevelHttpActionTest.java b/server/sonar-ce/src/test/java/org/sonar/ce/logging/ChangeLogLevelHttpActionTest.java
index 88435a2a8ba..40a1a5ce66f 100644
--- a/server/sonar-ce/src/test/java/org/sonar/ce/logging/ChangeLogLevelHttpActionTest.java
+++ b/server/sonar-ce/src/test/java/org/sonar/ce/logging/ChangeLogLevelHttpActionTest.java
@@ -26,8 +26,6 @@ import org.apache.commons.io.IOUtils;
 import org.junit.Test;
 import org.sonar.api.utils.log.LoggerLevel;
 import org.sonar.ce.httpd.HttpAction;
-import org.sonar.ce.log.CeProcessLogging;
-import org.sonar.db.Database;
 import org.sonar.server.platform.ServerLogging;
 
 import static fi.iki.elonen.NanoHTTPD.Method.GET;
@@ -42,9 +40,7 @@ import static org.sonar.ce.httpd.CeHttpUtils.createHttpSession;
 
 public class ChangeLogLevelHttpActionTest {
   private ServerLogging serverLogging = mock(ServerLogging.class);
-  private Database database = mock(Database.class);
-  private CeProcessLogging ceProcessLogging = new CeProcessLogging();
-  private ChangeLogLevelHttpAction underTest = new ChangeLogLevelHttpAction(serverLogging, database, ceProcessLogging);
+  private ChangeLogLevelHttpAction underTest = new ChangeLogLevelHttpAction(serverLogging);
 
   @Test
   public void register_to_path_changeLogLevel() {
@@ -78,42 +74,38 @@ public class ChangeLogLevelHttpActionTest {
   }
 
   @Test
-  public void changes_server_logging_and_disabled_database_logging_if_level_is_ERROR() {
+  public void changes_server_logging_if_level_is_ERROR() {
     NanoHTTPD.Response response = underTest.serve(createHttpSession(POST, ImmutableMap.of("level", "ERROR")));
 
     assertThat(response.getStatus()).isEqualTo(OK);
 
-    verify(serverLogging).changeLevel(ceProcessLogging, LoggerLevel.ERROR);
-    verify(database).enableSqlLogging(false);
+    verify(serverLogging).changeLevel(LoggerLevel.ERROR);
   }
 
   @Test
-  public void changes_server_logging_and_disabled_database_logging_if_level_is_INFO() {
+  public void changes_server_logging_if_level_is_INFO() {
     NanoHTTPD.Response response = underTest.serve(createHttpSession(POST, ImmutableMap.of("level", "INFO")));
 
     assertThat(response.getStatus()).isEqualTo(OK);
 
-    verify(serverLogging).changeLevel(ceProcessLogging, LoggerLevel.INFO);
-    verify(database).enableSqlLogging(false);
+    verify(serverLogging).changeLevel(LoggerLevel.INFO);
   }
 
   @Test
-  public void changes_server_logging_and_disabled_database_logging_if_level_is_DEBUG() {
+  public void changes_server_logging_if_level_is_DEBUG() {
     NanoHTTPD.Response response = underTest.serve(createHttpSession(POST, ImmutableMap.of("level", "DEBUG")));
 
     assertThat(response.getStatus()).isEqualTo(OK);
 
-    verify(serverLogging).changeLevel(ceProcessLogging, LoggerLevel.DEBUG);
-    verify(database).enableSqlLogging(false);
+    verify(serverLogging).changeLevel(LoggerLevel.DEBUG);
   }
 
   @Test
-  public void changes_server_logging_and_enable_database_logging_if_level_is_TRACE() {
+  public void changes_server_logging_if_level_is_TRACE() {
     NanoHTTPD.Response response = underTest.serve(createHttpSession(POST, ImmutableMap.of("level", "TRACE")));
 
     assertThat(response.getStatus()).isEqualTo(OK);
 
-    verify(serverLogging).changeLevel(ceProcessLogging, LoggerLevel.TRACE);
-    verify(database).enableSqlLogging(true);
+    verify(serverLogging).changeLevel(LoggerLevel.TRACE);
   }
 }
diff --git a/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/DistributedAnswer.java b/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/DistributedAnswer.java
index be6105cc511..f5a0bc920c8 100644
--- a/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/DistributedAnswer.java
+++ b/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/DistributedAnswer.java
@@ -28,6 +28,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * Answer of {@link DistributedCall}, aggregating the answers from
@@ -70,4 +71,21 @@ public class DistributedAnswer<T> {
   public void setFailed(Member member, Exception e) {
     failedMembers.put(member, e);
   }
+
+  public void propagateExceptions() {
+    if (!failedMembers.isEmpty()) {
+      String failedMemberNames = failedMembers.keySet().stream()
+        .map(m -> m.getStringAttribute(HazelcastMember.Attribute.NODE_NAME))
+        .collect(Collectors.joining(", "));
+      throw new IllegalStateException("Distributed cluster action in cluster nodes " + failedMemberNames + " (other nodes may have timed out)",
+        failedMembers.values().iterator().next());
+    }
+
+    if (!timedOutMembers.isEmpty()) {
+      String timedOutMemberNames = timedOutMembers.stream()
+        .map(m -> m.getStringAttribute(HazelcastMember.Attribute.NODE_NAME))
+        .collect(Collectors.joining(", "));
+      throw new IllegalStateException("Distributed cluster action timed out in cluster nodes " + timedOutMemberNames);
+    }
+  }
 }
diff --git a/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/HazelcastMemberSelectors.java b/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/HazelcastMemberSelectors.java
new file mode 100644
index 00000000000..3b7abe59f89
--- /dev/null
+++ b/server/sonar-process/src/main/java/org/sonar/process/cluster/hz/HazelcastMemberSelectors.java
@@ -0,0 +1,41 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.process.cluster.hz;
+
+import com.hazelcast.core.MemberSelector;
+import java.util.List;
+import org.sonar.process.ProcessId;
+
+import static java.util.Arrays.asList;
+import static org.sonar.process.ProcessId.fromKey;
+
+public class HazelcastMemberSelectors {
+
+  private HazelcastMemberSelectors() {
+  }
+
+  public static MemberSelector selectorForProcessIds(ProcessId... processIds) {
+    List<ProcessId> processIdList = asList(processIds);
+    return member -> {
+      ProcessId memberProcessId = fromKey(member.getStringAttribute(HazelcastMember.Attribute.PROCESS_KEY));
+      return processIdList.contains(memberProcessId);
+    };
+  }
+}
diff --git a/server/sonar-process/src/test/java/org/sonar/process/cluster/hz/HazelcastMemberSelectorsTest.java b/server/sonar-process/src/test/java/org/sonar/process/cluster/hz/HazelcastMemberSelectorsTest.java
new file mode 100644
index 00000000000..309e3463a32
--- /dev/null
+++ b/server/sonar-process/src/test/java/org/sonar/process/cluster/hz/HazelcastMemberSelectorsTest.java
@@ -0,0 +1,65 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.process.cluster.hz;
+
+import com.hazelcast.core.Member;
+import com.hazelcast.core.MemberSelector;
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.sonar.process.ProcessId.APP;
+import static org.sonar.process.ProcessId.COMPUTE_ENGINE;
+import static org.sonar.process.ProcessId.WEB_SERVER;
+import static org.sonar.process.cluster.hz.HazelcastMember.Attribute.PROCESS_KEY;
+
+public class HazelcastMemberSelectorsTest {
+
+  @Test
+  public void selecting_ce_nodes() throws Exception {
+    Member member = mock(Member.class);
+    MemberSelector underTest = HazelcastMemberSelectors.selectorForProcessIds(COMPUTE_ENGINE);
+
+    when(member.getStringAttribute(PROCESS_KEY)).thenReturn(COMPUTE_ENGINE.getKey());
+    assertThat(underTest.select(member)).isTrue();
+
+    when(member.getStringAttribute(PROCESS_KEY)).thenReturn(WEB_SERVER.getKey());
+    assertThat(underTest.select(member)).isFalse();
+
+    when(member.getStringAttribute(PROCESS_KEY)).thenReturn(APP.getKey());
+    assertThat(underTest.select(member)).isFalse();
+  }
+
+  @Test
+  public void selecting_web_and_app_nodes() throws Exception {
+    Member member = mock(Member.class);
+    MemberSelector underTest = HazelcastMemberSelectors.selectorForProcessIds(WEB_SERVER, APP);
+
+    when(member.getStringAttribute(PROCESS_KEY)).thenReturn(COMPUTE_ENGINE.getKey());
+    assertThat(underTest.select(member)).isFalse();
+
+    when(member.getStringAttribute(PROCESS_KEY)).thenReturn(WEB_SERVER.getKey());
+    assertThat(underTest.select(member)).isTrue();
+
+    when(member.getStringAttribute(PROCESS_KEY)).thenReturn(APP.getKey());
+    assertThat(underTest.select(member)).isTrue();
+  }
+}
\ No newline at end of file
diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/ServerLogging.java b/server/sonar-server/src/main/java/org/sonar/server/platform/ServerLogging.java
index a60631aed04..c7da8f50085 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/platform/ServerLogging.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/platform/ServerLogging.java
@@ -23,35 +23,60 @@ import ch.qos.logback.classic.Level;
 import ch.qos.logback.classic.Logger;
 import com.google.common.annotations.VisibleForTesting;
 import java.io.File;
+import org.picocontainer.Startable;
 import org.slf4j.LoggerFactory;
 import org.sonar.api.ce.ComputeEngineSide;
 import org.sonar.api.config.Configuration;
 import org.sonar.api.server.ServerSide;
 import org.sonar.api.utils.log.LoggerLevel;
 import org.sonar.api.utils.log.Loggers;
+import org.sonar.db.Database;
 import org.sonar.process.ProcessProperties;
 import org.sonar.process.logging.LogbackHelper;
 import org.sonar.server.app.ServerProcessLogging;
 
+import static org.sonar.api.utils.log.LoggerLevel.TRACE;
+
 @ServerSide
 @ComputeEngineSide
-public class ServerLogging {
+public class ServerLogging implements Startable {
 
+  /** Used for Hazelcast's distributed queries in cluster mode */
+  private static ServerLogging INSTANCE;
   private final LogbackHelper helper;
   private final Configuration config;
+  private final ServerProcessLogging serverProcessLogging;
+  private final Database database;
 
-  public ServerLogging(Configuration config) {
-    this(new LogbackHelper(), config);
+  public ServerLogging(Configuration config, ServerProcessLogging serverProcessLogging, Database database) {
+    this(new LogbackHelper(), config, serverProcessLogging, database);
   }
 
   @VisibleForTesting
-  ServerLogging(LogbackHelper helper, Configuration config) {
+  ServerLogging(LogbackHelper helper, Configuration config, ServerProcessLogging serverProcessLogging, Database database) {
     this.helper = helper;
     this.config = config;
+    this.serverProcessLogging = serverProcessLogging;
+    this.database = database;
+  }
+
+  @Override
+  public void start() {
+    INSTANCE = this;
+  }
+
+  @Override
+  public void stop() {
+    INSTANCE = null;
+  }
+
+  public static void changeLevelFromHazelcastDistributedQuery(LoggerLevel level) {
+    INSTANCE.changeLevel(level);
   }
 
-  public void changeLevel(ServerProcessLogging serverProcessLogging, LoggerLevel level) {
+  public void changeLevel(LoggerLevel level) {
     Level logbackLevel = Level.toLevel(level.name());
+    database.enableSqlLogging(level == TRACE);
     helper.changeRoot(serverProcessLogging.getLogLevelConfig(), logbackLevel);
     LoggerFactory.getLogger(ServerLogging.class).info("Level of logs changed to {}", level);
   }
diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel.java b/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel.java
index cd09c49490b..61bea5df8be 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel.java
@@ -38,6 +38,7 @@ public abstract class PlatformLevel {
   private final ComponentContainer container;
   private AddIfStartupLeader addIfStartupLeader;
   private AddIfCluster addIfCluster;
+  private AddIfStandalone addIfStandalone;
 
   public PlatformLevel(String name) {
     this.name = name;
@@ -156,6 +157,19 @@ public abstract class PlatformLevel {
     return addIfCluster;
   }
 
+  /**
+   * Add a component to container only if this is a standalone instance, without clustering.
+   *
+   * @throws IllegalStateException if called from PlatformLevel1, when cluster settings are not loaded
+   */
+  AddIfStandalone addIfStandalone(Object... objects) {
+    if (addIfStandalone == null) {
+      addIfStandalone = new AddIfStandalone(getWebServer().isStandalone());
+    }
+    addIfStandalone.ifAdd(objects);
+    return addIfStandalone;
+  }
+
   private WebServer getWebServer() {
     return getOptional(WebServer.class)
       .orElseThrow(() -> new IllegalStateException("WebServer not available in Pico yet"));
@@ -193,6 +207,12 @@ public abstract class PlatformLevel {
     }
   }
 
+  public final class AddIfStandalone extends AddIf {
+    private AddIfStandalone(boolean condition) {
+      super(condition);
+    }
+  }
+
   protected void addAll(Collection<?> objects) {
     add(objects.toArray(new Object[objects.size()]));
   }
diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java b/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java
index 5d015b81036..0ef1562a489 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java
@@ -112,6 +112,8 @@ import org.sonar.server.platform.monitoring.WebSystemInfoModule;
 import org.sonar.server.platform.web.WebPagesFilter;
 import org.sonar.server.platform.web.requestid.HttpRequestIdModule;
 import org.sonar.server.platform.ws.ChangeLogLevelAction;
+import org.sonar.server.platform.ws.ChangeLogLevelClusterService;
+import org.sonar.server.platform.ws.ChangeLogLevelStandaloneService;
 import org.sonar.server.platform.ws.DbMigrationStatusAction;
 import org.sonar.server.platform.ws.HealthActionModule;
 import org.sonar.server.platform.ws.L10nWs;
@@ -243,7 +245,10 @@ public class PlatformLevel4 extends PlatformLevel {
 
     addIfCluster(
       StartableHazelcastMember.class,
-      NodeHealthModule.class);
+      NodeHealthModule.class,
+      ChangeLogLevelClusterService.class);
+    addIfStandalone(
+      ChangeLogLevelStandaloneService.class);
 
     add(
       PluginDownloader.class,
diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/ws/ChangeLogLevelAction.java b/server/sonar-server/src/main/java/org/sonar/server/platform/ws/ChangeLogLevelAction.java
index 9bc42601479..a62a78962ec 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/platform/ws/ChangeLogLevelAction.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/platform/ws/ChangeLogLevelAction.java
@@ -23,10 +23,6 @@ import org.sonar.api.server.ws.Request;
 import org.sonar.api.server.ws.Response;
 import org.sonar.api.server.ws.WebService;
 import org.sonar.api.utils.log.LoggerLevel;
-import org.sonar.ce.http.CeHttpClient;
-import org.sonar.db.Database;
-import org.sonar.server.app.WebServerProcessLogging;
-import org.sonar.server.platform.ServerLogging;
 import org.sonar.server.user.UserSession;
 
 import static org.sonar.process.logging.LogbackHelper.allowedLogLevels;
@@ -36,17 +32,11 @@ public class ChangeLogLevelAction implements SystemWsAction {
   private static final String PARAM_LEVEL = "level";
 
   private final UserSession userSession;
-  private final ServerLogging logging;
-  private final Database db;
-  private final CeHttpClient ceHttpClient;
-  private final WebServerProcessLogging webServerProcessLogging;
+  private final ChangeLogLevelService service;
 
-  public ChangeLogLevelAction(UserSession userSession, ServerLogging logging, Database db, CeHttpClient ceHttpClient, WebServerProcessLogging webServerProcessLogging) {
+  public ChangeLogLevelAction(UserSession userSession, ChangeLogLevelService service) {
     this.userSession = userSession;
-    this.logging = logging;
-    this.db = db;
-    this.ceHttpClient = ceHttpClient;
-    this.webServerProcessLogging = webServerProcessLogging;
+    this.service = service;
   }
 
   @Override
@@ -69,9 +59,7 @@ public class ChangeLogLevelAction implements SystemWsAction {
     userSession.checkIsSystemAdministrator();
 
     LoggerLevel level = LoggerLevel.valueOf(wsRequest.mandatoryParam(PARAM_LEVEL));
-    db.enableSqlLogging(level.equals(LoggerLevel.TRACE));
-    logging.changeLevel(webServerProcessLogging, level);
-    ceHttpClient.changeLogLevel(level);
+    service.changeLogLevel(level);
     wsResponse.noContent();
   }
 }
diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/ws/ChangeLogLevelClusterService.java b/server/sonar-server/src/main/java/org/sonar/server/platform/ws/ChangeLogLevelClusterService.java
new file mode 100644
index 00000000000..362fa62114e
--- /dev/null
+++ b/server/sonar-server/src/main/java/org/sonar/server/platform/ws/ChangeLogLevelClusterService.java
@@ -0,0 +1,62 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.platform.ws;
+
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.LoggerLevel;
+import org.sonar.api.utils.log.Loggers;
+import org.sonar.process.ProcessId;
+import org.sonar.process.cluster.hz.DistributedCall;
+import org.sonar.process.cluster.hz.HazelcastMember;
+import org.sonar.process.cluster.hz.HazelcastMemberSelectors;
+import org.sonar.server.platform.ServerLogging;
+
+public class ChangeLogLevelClusterService implements ChangeLogLevelService {
+
+  private static final long CLUSTER_TIMEOUT_MILLIS = 5000;
+  private static final Logger LOGGER = Loggers.get(ChangeLogLevelClusterService.class);
+
+  private final HazelcastMember member;
+
+  public ChangeLogLevelClusterService(HazelcastMember member) {
+    this.member = member;
+  }
+
+  public void changeLogLevel(LoggerLevel level) {
+    try {
+      member.call(setLogLevelForNode(level), HazelcastMemberSelectors.selectorForProcessIds(ProcessId.WEB_SERVER, ProcessId.COMPUTE_ENGINE), CLUSTER_TIMEOUT_MILLIS)
+        .propagateExceptions();
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+    }
+  }
+
+  private static DistributedCall<Object> setLogLevelForNode(LoggerLevel level) {
+    return () -> {
+      try {
+        ServerLogging.changeLevelFromHazelcastDistributedQuery(level);
+      } catch (Exception e) {
+        LOGGER.error("Setting log level to '" + level.name() + "' in this cluster node failed", e);
+        throw new IllegalStateException("Setting log level to '" + level.name() + "' in this cluster node failed", e);
+      }
+      return null;
+    };
+  }
+}
diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/ws/ChangeLogLevelService.java b/server/sonar-server/src/main/java/org/sonar/server/platform/ws/ChangeLogLevelService.java
new file mode 100644
index 00000000000..1ef59c76b7d
--- /dev/null
+++ b/server/sonar-server/src/main/java/org/sonar/server/platform/ws/ChangeLogLevelService.java
@@ -0,0 +1,28 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.platform.ws;
+
+import org.sonar.api.utils.log.LoggerLevel;
+
+public interface ChangeLogLevelService {
+
+  void changeLogLevel(LoggerLevel level);
+
+}
diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/ws/ChangeLogLevelStandaloneService.java b/server/sonar-server/src/main/java/org/sonar/server/platform/ws/ChangeLogLevelStandaloneService.java
new file mode 100644
index 00000000000..fc551a0f426
--- /dev/null
+++ b/server/sonar-server/src/main/java/org/sonar/server/platform/ws/ChangeLogLevelStandaloneService.java
@@ -0,0 +1,40 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.platform.ws;
+
+import org.sonar.api.utils.log.LoggerLevel;
+import org.sonar.ce.http.CeHttpClient;
+import org.sonar.server.platform.ServerLogging;
+
+public class ChangeLogLevelStandaloneService implements ChangeLogLevelService {
+
+  private final ServerLogging logging;
+  private final CeHttpClient ceHttpClient;
+
+  public ChangeLogLevelStandaloneService(ServerLogging logging, CeHttpClient ceHttpClient) {
+    this.logging = logging;
+    this.ceHttpClient = ceHttpClient;
+  }
+
+  public void changeLogLevel(LoggerLevel level) {
+    logging.changeLevel(level);
+    ceHttpClient.changeLogLevel(level);
+  }
+}
diff --git a/server/sonar-server/src/test/java/org/sonar/server/platform/ServerLoggingTest.java b/server/sonar-server/src/test/java/org/sonar/server/platform/ServerLoggingTest.java
index 05fec294c8b..440e7290fe7 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/platform/ServerLoggingTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/platform/ServerLoggingTest.java
@@ -34,6 +34,7 @@ import org.junit.runner.RunWith;
 import org.sonar.api.config.internal.MapSettings;
 import org.sonar.api.utils.log.LogTester;
 import org.sonar.api.utils.log.LoggerLevel;
+import org.sonar.db.Database;
 import org.sonar.process.ProcessProperties;
 import org.sonar.process.logging.LogLevelConfig;
 import org.sonar.process.logging.LogbackHelper;
@@ -41,9 +42,15 @@ import org.sonar.server.app.ServerProcessLogging;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
+import static org.sonar.api.utils.log.LoggerLevel.DEBUG;
+import static org.sonar.api.utils.log.LoggerLevel.ERROR;
+import static org.sonar.api.utils.log.LoggerLevel.INFO;
+import static org.sonar.api.utils.log.LoggerLevel.TRACE;
+import static org.sonar.api.utils.log.LoggerLevel.WARN;
 
 @RunWith(DataProviderRunner.class)
 public class ServerLoggingTest {
@@ -56,7 +63,9 @@ public class ServerLoggingTest {
   private final String rootLoggerName = RandomStringUtils.randomAlphabetic(20);
   private LogbackHelper logbackHelper = spy(new LogbackHelper());
   private MapSettings settings = new MapSettings();
-  private ServerLogging underTest = new ServerLogging(logbackHelper, settings.asConfig());
+  private final ServerProcessLogging serverProcessLogging = mock(ServerProcessLogging.class);
+  private final Database database = mock(Database.class);
+  private ServerLogging underTest = new ServerLogging(logbackHelper, settings.asConfig(), serverProcessLogging, database);
 
   @Rule
   public LogTester logTester = new LogTester();
@@ -71,28 +80,45 @@ public class ServerLoggingTest {
 
   @Test
   public void getRootLoggerLevel() {
-    logTester.setLevel(LoggerLevel.TRACE);
-    assertThat(underTest.getRootLoggerLevel()).isEqualTo(LoggerLevel.TRACE);
+    logTester.setLevel(TRACE);
+    assertThat(underTest.getRootLoggerLevel()).isEqualTo(TRACE);
   }
 
   @Test
   @UseDataProvider("supportedSonarApiLevels")
   public void changeLevel_calls_changeRoot_with_LogLevelConfig_and_level_converted_to_logback_class_then_log_INFO_message(LoggerLevel level) {
-    ServerProcessLogging serverProcessLogging = mock(ServerProcessLogging.class);
     LogLevelConfig logLevelConfig = LogLevelConfig.newBuilder(rootLoggerName).build();
     when(serverProcessLogging.getLogLevelConfig()).thenReturn(logLevelConfig);
 
-    underTest.changeLevel(serverProcessLogging, level);
+    underTest.changeLevel(level);
 
     verify(logbackHelper).changeRoot(logLevelConfig, Level.valueOf(level.name()));
   }
 
+  @Test
+  public void changeLevel_to_trace_enables_db_logging() {
+    LogLevelConfig logLevelConfig = LogLevelConfig.newBuilder(rootLoggerName).build();
+    when(serverProcessLogging.getLogLevelConfig()).thenReturn(logLevelConfig);
+
+    reset(database);
+    underTest.changeLevel(INFO);
+    verify(database).enableSqlLogging(false);
+
+    reset(database);
+    underTest.changeLevel(DEBUG);
+    verify(database).enableSqlLogging(false);
+
+    reset(database);
+    underTest.changeLevel(TRACE);
+    verify(database).enableSqlLogging(true);
+  }
+
   @DataProvider
   public static Object[][] supportedSonarApiLevels() {
     return new Object[][] {
-      {LoggerLevel.INFO},
-      {LoggerLevel.DEBUG},
-      {LoggerLevel.TRACE}
+      {INFO},
+      {DEBUG},
+      {TRACE}
     };
   }
 
@@ -101,7 +127,7 @@ public class ServerLoggingTest {
     expectedException.expect(IllegalArgumentException.class);
     expectedException.expectMessage("ERROR log level is not supported (allowed levels are [TRACE, DEBUG, INFO])");
 
-    underTest.changeLevel(mock(ServerProcessLogging.class), LoggerLevel.ERROR);
+    underTest.changeLevel(ERROR);
   }
 
   @Test
@@ -109,6 +135,6 @@ public class ServerLoggingTest {
     expectedException.expect(IllegalArgumentException.class);
     expectedException.expectMessage("WARN log level is not supported (allowed levels are [TRACE, DEBUG, INFO])");
 
-    underTest.changeLevel(mock(ServerProcessLogging.class), LoggerLevel.WARN);
+    underTest.changeLevel(WARN);
   }
 }
diff --git a/server/sonar-server/src/test/java/org/sonar/server/platform/platformlevel/PlatformLevelTest.java b/server/sonar-server/src/test/java/org/sonar/server/platform/platformlevel/PlatformLevelTest.java
index ae60e8dbdb0..8f039e9f411 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/platform/platformlevel/PlatformLevelTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/platform/platformlevel/PlatformLevelTest.java
@@ -72,4 +72,20 @@ public class PlatformLevelTest {
     PlatformLevel.AddIfCluster addIfCluster = underTest.addIfCluster();
     IntStream.range(0, 1 + new Random().nextInt(4)).forEach(i -> assertThat(underTest.addIfCluster()).isSameAs(addIfCluster));
   }
+
+  @Test
+  public void addIfStandalone_throws_ISE_if_container_does_not_have_WebServer_object() {
+    expectedException.expect(IllegalStateException.class);
+    expectedException.expectMessage("WebServer not available in Pico yet");
+
+    underTest.addIfCluster();
+  }
+
+  @Test
+  public void addIfStandalone_always_returns_the_same_instance() {
+    underTest.add(Mockito.mock(WebServer.class));
+
+    PlatformLevel.AddIfCluster addIfCluster = underTest.addIfCluster();
+    IntStream.range(0, 1 + new Random().nextInt(4)).forEach(i -> assertThat(underTest.addIfCluster()).isSameAs(addIfCluster));
+  }
 }
diff --git a/server/sonar-server/src/test/java/org/sonar/server/platform/ws/ChangeLogLevelActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/platform/ws/ChangeLogLevelActionTest.java
index 3650dc10d7b..b1931740492 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/platform/ws/ChangeLogLevelActionTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/platform/ws/ChangeLogLevelActionTest.java
@@ -25,8 +25,6 @@ import org.junit.rules.ExpectedException;
 import org.sonar.api.utils.log.LoggerLevel;
 import org.sonar.ce.http.CeHttpClient;
 import org.sonar.ce.http.CeHttpClientImpl;
-import org.sonar.db.Database;
-import org.sonar.server.app.WebServerProcessLogging;
 import org.sonar.server.exceptions.ForbiddenException;
 import org.sonar.server.platform.ServerLogging;
 import org.sonar.server.tester.UserSessionRule;
@@ -43,10 +41,8 @@ public class ChangeLogLevelActionTest {
   public ExpectedException expectedException = ExpectedException.none();
 
   private ServerLogging serverLogging = mock(ServerLogging.class);
-  private Database db = mock(Database.class);
   private CeHttpClient ceHttpClient = mock(CeHttpClientImpl.class);
-  private WebServerProcessLogging webServerProcessLogging = new WebServerProcessLogging();
-  private ChangeLogLevelAction underTest = new ChangeLogLevelAction(userSession, serverLogging, db, ceHttpClient, webServerProcessLogging);
+  private ChangeLogLevelAction underTest = new ChangeLogLevelAction(userSession, new ChangeLogLevelStandaloneService(serverLogging, ceHttpClient));
   private WsActionTester actionTester = new WsActionTester(underTest);
 
   @Test
@@ -74,9 +70,8 @@ public class ChangeLogLevelActionTest {
       .setMethod("POST")
       .execute();
 
-    verify(serverLogging).changeLevel(webServerProcessLogging, LoggerLevel.DEBUG);
+    verify(serverLogging).changeLevel(LoggerLevel.DEBUG);
     verify(ceHttpClient).changeLogLevel(LoggerLevel.DEBUG);
-    verify(db).enableSqlLogging(false);
   }
 
   @Test
@@ -88,9 +83,8 @@ public class ChangeLogLevelActionTest {
       .setMethod("POST")
       .execute();
 
-    verify(serverLogging).changeLevel(webServerProcessLogging, LoggerLevel.TRACE);
+    verify(serverLogging).changeLevel(LoggerLevel.TRACE);
     verify(ceHttpClient).changeLogLevel(LoggerLevel.TRACE);
-    verify(db).enableSqlLogging(true);
   }
 
   @Test
diff --git a/sonar-ws/src/main/java/org/sonarqube/ws/client/system/SystemService.java b/sonar-ws/src/main/java/org/sonarqube/ws/client/system/SystemService.java
index cb8c55aec61..916072d2157 100644
--- a/sonar-ws/src/main/java/org/sonarqube/ws/client/system/SystemService.java
+++ b/sonar-ws/src/main/java/org/sonarqube/ws/client/system/SystemService.java
@@ -24,6 +24,7 @@ import org.sonarqube.ws.client.BaseService;
 import org.sonarqube.ws.client.GetRequest;
 import org.sonarqube.ws.client.PostRequest;
 import org.sonarqube.ws.client.WsConnector;
+import org.sonarqube.ws.client.WsResponse;
 
 public class SystemService extends BaseService {
   public SystemService(WsConnector wsConnector) {
@@ -41,4 +42,12 @@ public class SystemService extends BaseService {
   public WsSystem.StatusResponse status() {
     return call(new GetRequest(path("status")), WsSystem.StatusResponse.parser());
   }
+
+  public void changeLogLevel(String level) {
+    call(new PostRequest(path("change_log_level")).setParam("level", level));
+  }
+
+  public WsResponse info() {
+    return call(new GetRequest(path("info")));
+  }
 }
diff --git a/sonar-ws/src/test/java/org/sonarqube/ws/client/system/SystemServiceTest.java b/sonar-ws/src/test/java/org/sonarqube/ws/client/system/SystemServiceTest.java
index a473bda635e..145642a5278 100644
--- a/sonar-ws/src/test/java/org/sonarqube/ws/client/system/SystemServiceTest.java
+++ b/sonar-ws/src/test/java/org/sonarqube/ws/client/system/SystemServiceTest.java
@@ -64,4 +64,25 @@ public class SystemServiceTest {
         .hasPath("restart")
         .andNoOtherParam();
   }
+
+  @Test
+  public void test_changeLogLevel() throws Exception {
+    underTest.changeLogLevel("TRACE");
+
+    PostRequest postRequest = serviceTester.getPostRequest();
+    serviceTester.assertThat(postRequest)
+        .hasPath("change_log_level")
+        .hasParam("level", "TRACE")
+        .andNoOtherParam();
+  }
+
+  @Test
+  public void test_info() throws Exception {
+    underTest.info();
+
+    GetRequest getRequest = serviceTester.getGetRequest();
+    serviceTester.assertThat(getRequest)
+        .hasPath("info")
+        .andNoOtherParam();
+  }
 }
diff --git a/tests/src/test/java/org/sonarqube/tests/cluster/ClusterTest.java b/tests/src/test/java/org/sonarqube/tests/cluster/ClusterTest.java
index 2fab9bedb0b..a6afc2ca1fb 100644
--- a/tests/src/test/java/org/sonarqube/tests/cluster/ClusterTest.java
+++ b/tests/src/test/java/org/sonarqube/tests/cluster/ClusterTest.java
@@ -19,12 +19,14 @@
  */
 package org.sonarqube.tests.cluster;
 
+import com.google.gson.internal.LinkedTreeMap;
 import com.sonar.orchestrator.Orchestrator;
 import com.sonar.orchestrator.OrchestratorBuilder;
 import com.sonar.orchestrator.db.DefaultDatabase;
 import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.function.BinaryOperator;
 import java.util.function.Consumer;
 import java.util.stream.IntStream;
@@ -38,8 +40,9 @@ import org.junit.rules.TemporaryFolder;
 import org.junit.rules.TestRule;
 import org.junit.rules.Timeout;
 import org.sonarqube.ws.WsSystem;
-import org.sonarqube.ws.client.GetRequest;
 import org.sonarqube.ws.client.HttpException;
+import org.sonarqube.ws.client.issue.SearchWsRequest;
+import util.ItUtils;
 
 import static com.google.common.base.Preconditions.checkState;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -70,24 +73,6 @@ public class ClusterTest {
     db.stop();
   }
 
-  /**
-   * TODO WIP
-   */
-  @Test
-  public void wip() throws Exception {
-    try (Cluster cluster = newCluster(3, 2)) {
-      cluster.getNodes().forEach(Node::start);
-
-      Node app = cluster.getAppNode(0);
-      app.waitForHealthGreen();
-
-      System.out.println("-----------------------------------------------------------------------");
-      String json = app.wsClient().wsConnector().call(new GetRequest("api/system/info")).content();
-      System.out.println(json);
-      System.out.println("-----------------------------------------------------------------------");
-    }
-  }
-
   @Test
   public void test_high_availability_topology() throws Exception {
     try (Cluster cluster = newCluster(3, 2)) {
@@ -265,6 +250,37 @@ public class ClusterTest {
   }
 
   @Test
+  public void set_log_level_affects_all_nodes() throws Exception {
+    try (Cluster cluster = newCluster(2, 2)) {
+      cluster.getNodes().forEach(Node::start);
+      cluster.getAppNodes().forEach(Node::waitForStatusUp);
+
+      cluster.getAppNodes().forEach(node -> {
+        assertThat(node.webLogsContain(" TRACE web[")).isFalse();
+      });
+
+      cluster.getAppNode(0).wsClient().system().changeLogLevel("TRACE");
+
+      cluster.getAppNodes().forEach(node -> {
+
+        // do something, that will produce logging
+        node.wsClient().issues().search(new SearchWsRequest());
+
+        // check logs
+        assertThat(node.webLogsContain(" TRACE web[")).isTrue();
+      });
+
+      Map<String, Object> data = ItUtils.jsonToMap(cluster.getAppNode(0).wsClient().system().info().content());
+      ArrayList<Object> applicationNodes = (ArrayList<Object>) data.get("Application Nodes");
+      applicationNodes.forEach(node -> {
+        LinkedTreeMap<Object, Object> nodeData = (LinkedTreeMap<Object, Object>) node;
+        LinkedTreeMap<Object, Object> ceLoggingData = (LinkedTreeMap<Object, Object>) nodeData.get("Compute Engine Logging");
+        assertThat(ceLoggingData.get("Logs Level")).as("Compute engine logs level of a node").isEqualTo("TRACE");
+      });
+    }
+  }
+
+  @Test
   public void restart_action_is_not_allowed_for_cluster_nodes() throws Exception {
     try (Cluster cluster = newCluster(2, 1)) {
       cluster.getNodes().forEach(Node::start);
diff --git a/tests/src/test/java/org/sonarqube/tests/cluster/Node.java b/tests/src/test/java/org/sonarqube/tests/cluster/Node.java
index e8846c28042..5b684ac4a24 100644
--- a/tests/src/test/java/org/sonarqube/tests/cluster/Node.java
+++ b/tests/src/test/java/org/sonarqube/tests/cluster/Node.java
@@ -207,7 +207,7 @@ class Node {
     return content.hasText(message);
   }
 
-  private boolean webLogsContain(String message) {
+  boolean webLogsContain(String message) {
     if (orchestrator.getServer() == null) {
       return false;
     }