SONAR-9741 app shares search node info in hazelcast

12 files changed, 631 insertions, 7 deletions

diff --git a/server/sonar-main/src/main/java/org/sonar/application/SchedulerImpl.java b/server/sonar-main/src/main/java/org/sonar/application/SchedulerImpl.java
index 7311297dd3c..621cce82b3b 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/SchedulerImpl.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/SchedulerImpl.java
@@ -28,18 +28,23 @@ import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.sonar.api.utils.System2;
+import org.sonar.application.cluster.ClusterAppState;
 import org.sonar.application.config.AppSettings;
 import org.sonar.application.config.ClusterSettings;
-import org.sonar.process.command.CommandFactory;
-import org.sonar.process.command.EsCommand;
-import org.sonar.process.command.JavaCommand;
-import org.sonar.application.process.ProcessLauncher;
+import org.sonar.application.health.HealthStateSharing;
+import org.sonar.application.health.HealthStateSharingImpl;
+import org.sonar.application.health.SearchNodeHealthProvider;
 import org.sonar.application.process.Lifecycle;
 import org.sonar.application.process.ProcessEventListener;
+import org.sonar.application.process.ProcessLauncher;
 import org.sonar.application.process.ProcessLifecycleListener;
 import org.sonar.application.process.ProcessMonitor;
 import org.sonar.application.process.SQProcess;
 import org.sonar.process.ProcessId;
+import org.sonar.process.command.CommandFactory;
+import org.sonar.process.command.EsCommand;
+import org.sonar.process.command.JavaCommand;
 
 public class SchedulerImpl implements Scheduler, ProcessEventListener, ProcessLifecycleListener, AppStateListener {
 
@@ -60,6 +65,7 @@ public class SchedulerImpl implements Scheduler, ProcessEventListener, ProcessLi
   private final AtomicInteger stopCountDown = new AtomicInteger(0);
   private StopperThread stopperThread;
   private RestarterThread restarterThread;
+  private HealthStateSharing healthStateSharing;
   private long processWatcherDelayMs = SQProcess.DEFAULT_WATCHER_DELAY_MS;
 
   public SchedulerImpl(AppSettings settings, AppReloader appReloader, CommandFactory commandFactory,
@@ -99,6 +105,7 @@ public class SchedulerImpl implements Scheduler, ProcessEventListener, ProcessLi
   }
 
   private void tryToStartAll() {
+    tryToStartHealthStateSharing();
     tryToStartEs();
     tryToStartWeb();
     tryToStartCe();
@@ -137,6 +144,18 @@ public class SchedulerImpl implements Scheduler, ProcessEventListener, ProcessLi
     }
   }
 
+  private void tryToStartHealthStateSharing() {
+    if (healthStateSharing == null
+      && appState instanceof ClusterAppState
+      && ClusterSettings.isLocalElasticsearchEnabled(settings)) {
+      ClusterAppState clusterAppState = (ClusterAppState) appState;
+      this.healthStateSharing = new HealthStateSharingImpl(
+        clusterAppState.getHazelcastClient(),
+        new SearchNodeHealthProvider(settings.getProps(), System2.INSTANCE));
+      this.healthStateSharing.start();
+    }
+  }
+
   private boolean isEsClientStartable() {
     boolean requireLocalEs = ClusterSettings.isLocalElasticsearchEnabled(settings);
     return appState.isOperational(ProcessId.ELASTICSEARCH, requireLocalEs);
@@ -171,6 +190,7 @@ public class SchedulerImpl implements Scheduler, ProcessEventListener, ProcessLi
     stopProcess(ProcessId.COMPUTE_ENGINE);
     stopProcess(ProcessId.WEB_SERVER);
     stopProcess(ProcessId.ELASTICSEARCH);
+    stopHealthStateSharing();
   }
 
   /**
@@ -184,6 +204,12 @@ public class SchedulerImpl implements Scheduler, ProcessEventListener, ProcessLi
     }
   }
 
+  private void stopHealthStateSharing() {
+    if (healthStateSharing != null) {
+      healthStateSharing.stop();
+    }
+  }
+
   /**
    * Blocks until all processes are stopped. Pending restart, if
    * any, is disabled.
diff --git a/server/sonar-main/src/main/java/org/sonar/application/cluster/AppStateClusterImpl.java b/server/sonar-main/src/main/java/org/sonar/application/cluster/AppStateClusterImpl.java
index 498ef00ef12..717f41b287b 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/cluster/AppStateClusterImpl.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/cluster/AppStateClusterImpl.java
@@ -27,16 +27,16 @@ import java.util.stream.Collectors;
 import javax.annotation.Nonnull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.sonar.application.AppState;
 import org.sonar.application.AppStateListener;
 import org.sonar.application.config.AppSettings;
+import org.sonar.cluster.localclient.HazelcastClient;
 import org.sonar.process.ProcessId;
 
 import static org.sonar.cluster.ClusterProperties.CLUSTER_ENABLED;
 import static org.sonar.cluster.ClusterProperties.CLUSTER_LOCALENDPOINT;
 import static org.sonar.cluster.ClusterProperties.CLUSTER_MEMBERUUID;
 
-public class AppStateClusterImpl implements AppState {
+public class AppStateClusterImpl implements ClusterAppState {
   private static Logger LOGGER = LoggerFactory.getLogger(AppStateClusterImpl.class);
 
   private final Map<ProcessId, Boolean> localProcesses = new EnumMap<>(ProcessId.class);
@@ -112,6 +112,11 @@ public class AppStateClusterImpl implements AppState {
     return hazelcastCluster;
   }
 
+  @Override
+  public HazelcastClient getHazelcastClient() {
+    return hazelcastCluster.getHazelcastClient();
+  }
+
   /**
    * Only used for testing purpose
    *
diff --git a/server/sonar-main/src/main/java/org/sonar/application/cluster/ClusterAppState.java b/server/sonar-main/src/main/java/org/sonar/application/cluster/ClusterAppState.java
new file mode 100644
index 00000000000..044fcb99fd4
--- /dev/null
+++ b/server/sonar-main/src/main/java/org/sonar/application/cluster/ClusterAppState.java
@@ -0,0 +1,27 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.application.cluster;
+
+import org.sonar.application.AppState;
+import org.sonar.cluster.localclient.HazelcastClient;
+
+public interface ClusterAppState extends AppState {
+  HazelcastClient getHazelcastClient();
+}
diff --git a/server/sonar-main/src/main/java/org/sonar/application/cluster/HazelcastCluster.java b/server/sonar-main/src/main/java/org/sonar/application/cluster/HazelcastCluster.java
index cea75088e0a..b107751c055 100644
--- a/server/sonar-main/src/main/java/org/sonar/application/cluster/HazelcastCluster.java
+++ b/server/sonar-main/src/main/java/org/sonar/application/cluster/HazelcastCluster.java
@@ -43,10 +43,14 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.locks.Lock;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.NetworkUtils;
 import org.sonar.application.AppStateListener;
+import org.sonar.cluster.ClusterObjectKeys;
+import org.sonar.cluster.localclient.HazelcastClient;
 import org.sonar.process.MessageException;
 import org.sonar.process.NodeType;
 import org.sonar.process.ProcessId;
@@ -277,6 +281,53 @@ public class HazelcastCluster implements AutoCloseable {
     return format("%s:%d", localAddress.getHost(), localAddress.getPort());
   }
 
+  public HazelcastClient getHazelcastClient() {
+    return new HazelcastInstanceClient(hzInstance);
+  }
+
+  private static class HazelcastInstanceClient implements HazelcastClient {
+    private final HazelcastInstance hzInstance;
+
+    private HazelcastInstanceClient(HazelcastInstance hzInstance) {
+      this.hzInstance = hzInstance;
+    }
+
+    @Override
+    public <E> Set<E> getSet(String s) {
+      return hzInstance.getSet(s);
+    }
+
+    @Override
+    public <E> List<E> getList(String s) {
+      return hzInstance.getList(s);
+    }
+
+    @Override
+    public <K, V> Map<K, V> getMap(String s) {
+      return hzInstance.getMap(s);
+    }
+
+    @Override
+    public <K, V> Map<K, V> getReplicatedMap(String s) {
+      return hzInstance.getReplicatedMap(s);
+    }
+
+    @Override
+    public String getClientUUID() {
+      return hzInstance.getLocalEndpoint().getUuid();
+    }
+
+    @Override
+    public Set<String> getConnectedClients() {
+      return hzInstance.getSet(ClusterObjectKeys.CLIENT_UUIDS);
+    }
+
+    @Override
+    public Lock getLock(String s) {
+      return hzInstance.getLock(s);
+    }
+  }
+
   private class OperationalProcessListener implements EntryListener<ClusterProcess, Boolean> {
     @Override
     public void entryAdded(EntryEvent<ClusterProcess, Boolean> event) {
diff --git a/server/sonar-main/src/main/java/org/sonar/application/health/DelegateHealthStateRefresherExecutorService.java b/server/sonar-main/src/main/java/org/sonar/application/health/DelegateHealthStateRefresherExecutorService.java
new file mode 100644
index 00000000000..f47bce53a3b
--- /dev/null
+++ b/server/sonar-main/src/main/java/org/sonar/application/health/DelegateHealthStateRefresherExecutorService.java
@@ -0,0 +1,124 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.application.health;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import org.sonar.cluster.health.HealthStateRefresherExecutorService;
+
+class DelegateHealthStateRefresherExecutorService implements HealthStateRefresherExecutorService {
+  private final ScheduledExecutorService delegate;
+
+  DelegateHealthStateRefresherExecutorService(ScheduledExecutorService delegate) {
+    this.delegate = delegate;
+  }
+
+  @Override
+  public ScheduledFuture<?> schedule(Runnable command, long delay, TimeUnit unit) {
+    return delegate.schedule(command, delay, unit);
+  }
+
+  @Override
+  public <V> ScheduledFuture<V> schedule(Callable<V> callable, long delay, TimeUnit unit) {
+    return delegate.schedule(callable, delay, unit);
+  }
+
+  @Override
+  public ScheduledFuture<?> scheduleAtFixedRate(Runnable command, long initialDelay, long period, TimeUnit unit) {
+    return delegate.scheduleAtFixedRate(command, initialDelay, period, unit);
+  }
+
+  @Override
+  public ScheduledFuture<?> scheduleWithFixedDelay(Runnable command, long initialDelay, long delay, TimeUnit unit) {
+    return delegate.scheduleWithFixedDelay(command, initialDelay, delay, unit);
+  }
+
+  @Override
+  public void shutdown() {
+    delegate.shutdown();
+  }
+
+  @Override
+  public List<Runnable> shutdownNow() {
+    return delegate.shutdownNow();
+  }
+
+  @Override
+  public boolean isShutdown() {
+    return delegate.isShutdown();
+  }
+
+  @Override
+  public boolean isTerminated() {
+    return delegate.isTerminated();
+  }
+
+  @Override
+  public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
+    return delegate.awaitTermination(timeout, unit);
+  }
+
+  @Override
+  public <T> Future<T> submit(Callable<T> task) {
+    return delegate.submit(task);
+  }
+
+  @Override
+  public <T> Future<T> submit(Runnable task, T result) {
+    return delegate.submit(task, result);
+  }
+
+  @Override
+  public Future<?> submit(Runnable task) {
+    return delegate.submit(task);
+  }
+
+  @Override
+  public <T> List<Future<T>> invokeAll(Collection<? extends Callable<T>> tasks) throws InterruptedException {
+    return delegate.invokeAll(tasks);
+  }
+
+  @Override
+  public <T> List<Future<T>> invokeAll(Collection<? extends Callable<T>> tasks, long timeout, TimeUnit unit) throws InterruptedException {
+    return delegate.invokeAll(tasks, timeout, unit);
+  }
+
+  @Override
+  public <T> T invokeAny(Collection<? extends Callable<T>> tasks) throws InterruptedException, ExecutionException {
+    return delegate.invokeAny(tasks);
+  }
+
+  @Override
+  public <T> T invokeAny(Collection<? extends Callable<T>> tasks, long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
+    return delegate.invokeAny(tasks, timeout, unit);
+  }
+
+  @Override
+  public void execute(Runnable command) {
+    delegate.execute(command);
+  }
+}
diff --git a/server/sonar-main/src/main/java/org/sonar/application/health/HealthStateSharing.java b/server/sonar-main/src/main/java/org/sonar/application/health/HealthStateSharing.java
new file mode 100644
index 00000000000..0297ea8beed
--- /dev/null
+++ b/server/sonar-main/src/main/java/org/sonar/application/health/HealthStateSharing.java
@@ -0,0 +1,26 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.application.health;
+
+public interface HealthStateSharing {
+  void start();
+
+  void stop();
+}
diff --git a/server/sonar-main/src/main/java/org/sonar/application/health/HealthStateSharingImpl.java b/server/sonar-main/src/main/java/org/sonar/application/health/HealthStateSharingImpl.java
new file mode 100644
index 00000000000..06a24f59da5
--- /dev/null
+++ b/server/sonar-main/src/main/java/org/sonar/application/health/HealthStateSharingImpl.java
@@ -0,0 +1,88 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.application.health;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
+import org.sonar.cluster.health.HealthStateRefresher;
+import org.sonar.cluster.health.HealthStateRefresherExecutorService;
+import org.sonar.cluster.health.NodeHealthProvider;
+import org.sonar.cluster.health.SharedHealthStateImpl;
+import org.sonar.cluster.localclient.HazelcastClient;
+
+import static java.lang.String.format;
+
+public class HealthStateSharingImpl implements HealthStateSharing {
+  private static final Logger LOG = Loggers.get(HealthStateSharingImpl.class);
+
+  private final HazelcastClient hazelcastClient;
+  private final NodeHealthProvider nodeHealthProvider;
+  private HealthStateRefresherExecutorService executorService;
+  private HealthStateRefresher healthStateRefresher;
+
+  public HealthStateSharingImpl(HazelcastClient hazelcastClient, NodeHealthProvider nodeHealthProvider) {
+    this.hazelcastClient = hazelcastClient;
+    this.nodeHealthProvider = nodeHealthProvider;
+  }
+
+  @Override
+  public void start() {
+    executorService = new DelegateHealthStateRefresherExecutorService(
+      Executors.newSingleThreadScheduledExecutor(
+        new ThreadFactoryBuilder()
+          .setDaemon(false)
+          .setNameFormat("health_state_refresh-%d")
+          .build()));
+    healthStateRefresher = new HealthStateRefresher(executorService, nodeHealthProvider, new SharedHealthStateImpl(hazelcastClient));
+    healthStateRefresher.start();
+  }
+
+  @Override
+  public void stop() {
+    healthStateRefresher.stop();
+    stopExecutorService(executorService);
+  }
+
+  private static void stopExecutorService(ScheduledExecutorService executorService) {
+    // Disable new tasks from being submitted
+    executorService.shutdown();
+    try {
+      // Wait a while for existing tasks to terminate
+      if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) {
+        // Cancel currently executing tasks
+        executorService.shutdownNow();
+        // Wait a while for tasks to respond to being canceled
+        if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) {
+          LOG.warn(format("Pool %s did not terminate", HealthStateSharingImpl.class.getSimpleName()));
+        }
+      }
+    } catch (InterruptedException ie) {
+      LOG.warn(format("Termination of pool %s failed", HealthStateSharingImpl.class.getSimpleName()), ie);
+      // (Re-)Cancel if current thread also interrupted
+      executorService.shutdownNow();
+      Thread.currentThread().interrupt();
+    }
+  }
+
+}
diff --git a/server/sonar-main/src/main/java/org/sonar/application/health/SearchNodeHealthProvider.java b/server/sonar-main/src/main/java/org/sonar/application/health/SearchNodeHealthProvider.java
new file mode 100644
index 00000000000..83f424217e1
--- /dev/null
+++ b/server/sonar-main/src/main/java/org/sonar/application/health/SearchNodeHealthProvider.java
@@ -0,0 +1,57 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.application.health;
+
+import java.util.Random;
+import org.sonar.api.utils.System2;
+import org.sonar.cluster.ClusterProperties;
+import org.sonar.cluster.health.NodeDetails;
+import org.sonar.cluster.health.NodeHealth;
+import org.sonar.cluster.health.NodeHealthProvider;
+import org.sonar.process.Props;
+
+import static org.sonar.cluster.ClusterProperties.CLUSTER_NODE_PORT;
+
+public class SearchNodeHealthProvider implements NodeHealthProvider {
+  private final System2 system2;
+  private final NodeDetails nodeDetails;
+
+  public SearchNodeHealthProvider(Props props, System2 system2) {
+    this.system2 = system2;
+    this.nodeDetails = NodeDetails.newNodeDetailsBuilder()
+      .setType(NodeDetails.Type.SEARCH)
+      .setName(props.nonNullValue(ClusterProperties.CLUSTER_NAME) + new Random().nextInt(999))
+      // TODO read sonar.cluster.node.host
+      .setHost("host hardcoded for now")
+      .setPort(Integer.valueOf(props.nonNullValue(CLUSTER_NODE_PORT)))
+      // TODO is now good enough?
+      .setStarted(system2.now())
+      .build();
+  }
+
+  @Override
+  public NodeHealth get() {
+    return NodeHealth.newNodeHealthBuilder()
+      .setStatus(NodeHealth.Status.GREEN)
+      .setDetails(nodeDetails)
+      .setDate(system2.now())
+      .build();
+  }
+}
diff --git a/server/sonar-main/src/main/java/org/sonar/application/health/package-info.java b/server/sonar-main/src/main/java/org/sonar/application/health/package-info.java
new file mode 100644
index 00000000000..57ff328fbb1
--- /dev/null
+++ b/server/sonar-main/src/main/java/org/sonar/application/health/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+@ParametersAreNonnullByDefault
+package org.sonar.application.health;
+
+import javax.annotation.ParametersAreNonnullByDefault;
diff --git a/server/sonar-main/src/test/java/org/sonar/application/TestAppState.java b/server/sonar-main/src/test/java/org/sonar/application/TestAppState.java
index 27f6c4d5fb0..125ae867d40 100644
--- a/server/sonar-main/src/test/java/org/sonar/application/TestAppState.java
+++ b/server/sonar-main/src/test/java/org/sonar/application/TestAppState.java
@@ -19,6 +19,7 @@
  */
 package org.sonar.application;
 
+import com.google.common.base.Preconditions;
 import java.util.ArrayList;
 import java.util.EnumMap;
 import java.util.List;
@@ -27,14 +28,25 @@ import java.util.Optional;
 import java.util.concurrent.atomic.AtomicBoolean;
 import javax.annotation.Nonnull;
 import org.sonar.NetworkUtils;
+import org.sonar.application.cluster.ClusterAppState;
+import org.sonar.cluster.localclient.HazelcastClient;
 import org.sonar.process.ProcessId;
 
-public class TestAppState implements AppState {
+public class TestAppState implements ClusterAppState {
 
   private final Map<ProcessId, Boolean> localProcesses = new EnumMap<>(ProcessId.class);
   private final Map<ProcessId, Boolean> remoteProcesses = new EnumMap<>(ProcessId.class);
   private final List<AppStateListener> listeners = new ArrayList<>();
   private final AtomicBoolean webLeaderLocked = new AtomicBoolean(false);
+  private final HazelcastClient hazelcastClient;
+
+  public TestAppState() {
+    this(null);
+  }
+
+  public TestAppState(HazelcastClient hazelcastClient) {
+    this.hazelcastClient = hazelcastClient;
+  }
 
   @Override
   public void addListener(@Nonnull AppStateListener listener) {
@@ -88,6 +100,12 @@ public class TestAppState implements AppState {
   }
 
   @Override
+  public HazelcastClient getHazelcastClient() {
+    Preconditions.checkState(hazelcastClient != null, "An HazelcastClient should be provided when testing in cluster mode");
+    return hazelcastClient;
+  }
+
+  @Override
   public void close() {
     // nothing to do
   }
diff --git a/server/sonar-main/src/test/java/org/sonar/application/health/DelegateHealthStateRefresherExecutorServiceTest.java b/server/sonar-main/src/test/java/org/sonar/application/health/DelegateHealthStateRefresherExecutorServiceTest.java
new file mode 100644
index 00000000000..1b3dec9a0db
--- /dev/null
+++ b/server/sonar-main/src/test/java/org/sonar/application/health/DelegateHealthStateRefresherExecutorServiceTest.java
@@ -0,0 +1,153 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.application.health;
+
+import java.util.Collection;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.junit.Test;
+
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+public class DelegateHealthStateRefresherExecutorServiceTest {
+  private Random random = new Random();
+  private Runnable runnable = mock(Runnable.class);
+  private Callable callable = mock(Callable.class);
+  private Collection<Callable<Object>> callables = IntStream.range(0, random.nextInt(5))
+    .mapToObj(i -> (Callable<Object>) mock(Callable.class))
+    .collect(Collectors.toList());
+  private int initialDelay = random.nextInt(333);
+  private int delay = random.nextInt(333);
+  private int period = random.nextInt(333);
+  private int timeout = random.nextInt(333);
+  private Object result = new Object();
+  private ScheduledExecutorService executorService = mock(ScheduledExecutorService.class);
+  private DelegateHealthStateRefresherExecutorService underTest = new DelegateHealthStateRefresherExecutorService(executorService);
+
+  @Test
+  public void schedule() {
+    underTest.schedule(runnable, delay, SECONDS);
+
+    verify(executorService).schedule(runnable, delay, SECONDS);
+  }
+
+  @Test
+  public void schedule1() {
+    underTest.schedule(callable, delay, SECONDS);
+
+    verify(executorService).schedule(callable, delay, SECONDS);
+  }
+
+  @Test
+  public void scheduleAtFixedRate() {
+    underTest.scheduleAtFixedRate(runnable, initialDelay, period, SECONDS);
+    verify(executorService).scheduleAtFixedRate(runnable, initialDelay, period, SECONDS);
+  }
+
+  @Test
+  public void scheduleWithFixeddelay() {
+    underTest.scheduleWithFixedDelay(runnable, initialDelay, delay, TimeUnit.SECONDS);
+    verify(executorService).scheduleWithFixedDelay(runnable, initialDelay, delay, TimeUnit.SECONDS);
+  }
+
+  @Test
+  public void shutdown() {
+    underTest.shutdown();
+    verify(executorService).shutdown();
+  }
+
+  @Test
+  public void shutdownNow() {
+    underTest.shutdownNow();
+    verify(executorService).shutdownNow();
+  }
+
+  @Test
+  public void isShutdown() {
+    underTest.isShutdown();
+    verify(executorService).isShutdown();
+  }
+
+  @Test
+  public void isTerminated() {
+    underTest.isTerminated();
+    verify(executorService).isTerminated();
+  }
+
+  @Test
+  public void awaitTermination() throws InterruptedException {
+    underTest.awaitTermination(timeout, TimeUnit.SECONDS);
+
+    verify(executorService).awaitTermination(timeout, TimeUnit.SECONDS);
+  }
+
+  @Test
+  public void submit() {
+    underTest.submit(callable);
+
+    verify(executorService).submit(callable);
+  }
+
+  @Test
+  public void submit1() {
+    underTest.submit(runnable, result);
+
+    verify(executorService).submit(runnable, result);
+  }
+
+  @Test
+  public void submit2() {
+    underTest.submit(runnable);
+    verify(executorService).submit(runnable);
+  }
+
+  @Test
+  public void invokeAll() throws InterruptedException {
+    underTest.invokeAll(callables);
+    verify(executorService).invokeAll(callables);
+  }
+
+  @Test
+  public void invokeAll1() throws InterruptedException {
+    underTest.invokeAll(callables, timeout, SECONDS);
+    verify(executorService).invokeAll(callables, timeout, SECONDS);
+  }
+
+  @Test
+  public void invokeAny() throws InterruptedException, ExecutionException {
+    underTest.invokeAny(callables);
+    verify(executorService).invokeAny(callables);
+  }
+
+  @Test
+  public void invokeAny2() throws InterruptedException, ExecutionException, TimeoutException {
+    underTest.invokeAny(callables, timeout, SECONDS);
+    verify(executorService).invokeAny(callables, timeout, SECONDS);
+  }
+
+}
diff --git a/server/sonar-main/src/test/java/org/sonar/application/health/SearchNodeHealthProviderTest.java b/server/sonar-main/src/test/java/org/sonar/application/health/SearchNodeHealthProviderTest.java
new file mode 100644
index 00000000000..0bc57d85f38
--- /dev/null
+++ b/server/sonar-main/src/test/java/org/sonar/application/health/SearchNodeHealthProviderTest.java
@@ -0,0 +1,26 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.application.health;
+
+// TODO implement UT for SearchNodeHealthProviderTest when Daniel and Eric's branch in merged into master
+public class SearchNodeHealthProviderTest {
+
+
+}