SONAR-8935 Add a log when joining a cluster

7 files changed, 512 insertions, 211 deletions

diff --git a/server/sonar-process-monitor/src/main/java/org/sonar/application/SchedulerImpl.java b/server/sonar-process-monitor/src/main/java/org/sonar/application/SchedulerImpl.java
index 4cce5a17467..8359d66e08d 100644
--- a/server/sonar-process-monitor/src/main/java/org/sonar/application/SchedulerImpl.java
+++ b/server/sonar-process-monitor/src/main/java/org/sonar/application/SchedulerImpl.java
@@ -161,7 +161,6 @@ public class SchedulerImpl implements Scheduler, ProcessEventListener, ProcessLi
     if (process != null) {
       process.stop(1, TimeUnit.MINUTES);
     }
-
   }
 
   /**
diff --git a/server/sonar-process-monitor/src/main/java/org/sonar/application/cluster/AppStateClusterImpl.java b/server/sonar-process-monitor/src/main/java/org/sonar/application/cluster/AppStateClusterImpl.java
index 3f25a987408..cae4d880572 100644
--- a/server/sonar-process-monitor/src/main/java/org/sonar/application/cluster/AppStateClusterImpl.java
+++ b/server/sonar-process-monitor/src/main/java/org/sonar/application/cluster/AppStateClusterImpl.java
@@ -20,41 +20,22 @@
 
 package org.sonar.application.cluster;
 
-import com.hazelcast.config.Config;
-import com.hazelcast.config.JoinConfig;
-import com.hazelcast.config.NetworkConfig;
-import com.hazelcast.core.EntryEvent;
-import com.hazelcast.core.EntryListener;
-import com.hazelcast.core.Hazelcast;
-import com.hazelcast.core.HazelcastInstance;
-import com.hazelcast.core.IAtomicReference;
-import com.hazelcast.core.ILock;
-import com.hazelcast.core.MapEvent;
-import com.hazelcast.core.ReplicatedMap;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
 import java.util.EnumMap;
-import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 import javax.annotation.Nonnull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.sonar.application.AppState;
 import org.sonar.application.AppStateListener;
 import org.sonar.application.config.AppSettings;
 import org.sonar.process.ProcessId;
 
 public class AppStateClusterImpl implements AppState {
-  static final String OPERATIONAL_PROCESSES = "OPERATIONAL_PROCESSES";
-  static final String LEADER = "LEADER";
+  private static Logger LOGGER = LoggerFactory.getLogger(AppStateClusterImpl.class);
 
-  static final String SONARQUBE_VERSION = "SONARQUBE_VERSION";
-
-  private final List<AppStateListener> listeners = new ArrayList<>();
-  private final ReplicatedMap<ClusterProcess, Boolean> operationalProcesses;
   private final Map<ProcessId, Boolean> localProcesses = new EnumMap<>(ProcessId.class);
-  private final String listenerUuid;
-
-  final HazelcastInstance hzInstance;
+  private final HazelcastCluster hazelcastCluster;
 
   public AppStateClusterImpl(AppSettings appSettings) {
     ClusterProperties clusterProperties = new ClusterProperties(appSettings);
@@ -64,56 +45,15 @@ public class AppStateClusterImpl implements AppState {
       throw new IllegalStateException("Cluster is not enabled on this instance");
     }
 
-    Config hzConfig = new Config();
-    try {
-      hzConfig.setInstanceName(InetAddress.getLocalHost().getHostName());
-    } catch (UnknownHostException e) {
-      // Ignore it
-    }
-
-    hzConfig.getGroupConfig().setName(clusterProperties.getName());
-
-    // Configure the network instance
-    NetworkConfig netConfig = hzConfig.getNetworkConfig();
-    netConfig
-      .setPort(clusterProperties.getPort())
-      .setReuseAddress(true);
-
-    if (!clusterProperties.getNetworkInterfaces().isEmpty()) {
-      netConfig.getInterfaces()
-        .setEnabled(true)
-        .setInterfaces(clusterProperties.getNetworkInterfaces());
-    }
+    hazelcastCluster = HazelcastCluster.create(clusterProperties);
 
-    // Only allowing TCP/IP configuration
-    JoinConfig joinConfig = netConfig.getJoin();
-    joinConfig.getAwsConfig().setEnabled(false);
-    joinConfig.getMulticastConfig().setEnabled(false);
-    joinConfig.getTcpIpConfig().setEnabled(true);
-    joinConfig.getTcpIpConfig().setMembers(clusterProperties.getHosts());
-
-    // Tweak HazelCast configuration
-    hzConfig
-      // Increase the number of tries
-      .setProperty("hazelcast.tcp.join.port.try.count", "10")
-      // Don't bind on all interfaces
-      .setProperty("hazelcast.socket.bind.any", "false")
-      // Don't phone home
-      .setProperty("hazelcast.phone.home.enabled", "false")
-      // Use slf4j for logging
-      .setProperty("hazelcast.logging.type", "slf4j");
-
-    // We are not using the partition group of Hazelcast, so disabling it
-    hzConfig.getPartitionGroupConfig().setEnabled(false);
-
-    hzInstance = Hazelcast.newHazelcastInstance(hzConfig);
-    operationalProcesses = hzInstance.getReplicatedMap(OPERATIONAL_PROCESSES);
-    listenerUuid = operationalProcesses.addEntryListener(new OperationalProcessListener());
+    String members = hazelcastCluster.getMembers().stream().collect(Collectors.joining(","));
+    LOGGER.info("Joined the cluster [{}] that contains the following hosts : [{}]", hazelcastCluster.getName(), members);
   }
 
   @Override
   public void addListener(@Nonnull AppStateListener listener) {
-    listeners.add(listener);
+    hazelcastCluster.addListener(listener);
   }
 
   @Override
@@ -121,39 +61,18 @@ public class AppStateClusterImpl implements AppState {
     if (local) {
       return localProcesses.computeIfAbsent(processId, p -> false);
     }
-    for (Map.Entry<ClusterProcess, Boolean> entry : operationalProcesses.entrySet()) {
-      if (entry.getKey().getProcessId().equals(processId) && entry.getValue()) {
-        return true;
-      }
-    }
-    return false;
+    return hazelcastCluster.isOperational(processId);
   }
 
   @Override
   public void setOperational(@Nonnull ProcessId processId) {
     localProcesses.put(processId, true);
-    operationalProcesses.put(new ClusterProcess(getLocalUuid(), processId), Boolean.TRUE);
+    hazelcastCluster.setOperational(processId);
   }
 
   @Override
   public boolean tryToLockWebLeader() {
-    IAtomicReference<String> leader = hzInstance.getAtomicReference(LEADER);
-    if (leader.get() == null) {
-      ILock lock = hzInstance.getLock(LEADER);
-      lock.lock();
-      try {
-        if (leader.get() == null) {
-          leader.set(getLocalUuid());
-          return true;
-        } else {
-          return false;
-        }
-      } finally {
-        lock.unlock();
-      }
-    } else {
-      return false;
-    }
+    return hazelcastCluster.tryToLockWebLeader();
   }
 
   @Override
@@ -163,80 +82,25 @@ public class AppStateClusterImpl implements AppState {
 
   @Override
   public void close() {
-    if (hzInstance != null) {
-      operationalProcesses.removeEntryListener(listenerUuid);
-      operationalProcesses.keySet().forEach(
-        clusterNodeProcess -> {
-          if (clusterNodeProcess.getNodeUuid().equals(getLocalUuid())) {
-            operationalProcesses.remove(clusterNodeProcess);
-          }
-        });
-      hzInstance.shutdown();
-    }
+    hazelcastCluster.close();
   }
 
   @Override
   public void registerSonarQubeVersion(String sonarqubeVersion) {
-    IAtomicReference<String> sqVersion = hzInstance.getAtomicReference(SONARQUBE_VERSION);
-    if (sqVersion.get() == null) {
-      ILock lock = hzInstance.getLock(SONARQUBE_VERSION);
-      lock.lock();
-      try {
-        if (sqVersion.get() == null) {
-          sqVersion.set(sonarqubeVersion);
-        }
-      } finally {
-        lock.unlock();
-      }
-    }
-
-    String clusterVersion = sqVersion.get();
-    if (!sqVersion.get().equals(sonarqubeVersion)) {
-      hzInstance.shutdown();
-      throw new IllegalStateException(
-        String.format("The local version %s is not the same as the cluster %s", sonarqubeVersion, clusterVersion)
-      );
-    }
+    hazelcastCluster.registerSonarQubeVersion(sonarqubeVersion);
   }
 
-  private String getLocalUuid() {
-    return hzInstance.getLocalEndpoint().getUuid();
+  HazelcastCluster getHazelcastCluster() {
+    return hazelcastCluster;
   }
 
-  private class OperationalProcessListener implements EntryListener<ClusterProcess, Boolean> {
-
-    @Override
-    public void entryAdded(EntryEvent<ClusterProcess, Boolean> event) {
-      if (event.getValue()) {
-        listeners.forEach(appStateListener -> appStateListener.onAppStateOperational(event.getKey().getProcessId()));
-      }
-    }
-
-    @Override
-    public void entryRemoved(EntryEvent<ClusterProcess, Boolean> event) {
-      // Ignore it
-    }
-
-    @Override
-    public void entryUpdated(EntryEvent<ClusterProcess, Boolean> event) {
-      if (event.getValue()) {
-        listeners.forEach(appStateListener -> appStateListener.onAppStateOperational(event.getKey().getProcessId()));
-      }
-    }
-
-    @Override
-    public void entryEvicted(EntryEvent<ClusterProcess, Boolean> event) {
-      // Ignore it
-    }
-
-    @Override
-    public void mapCleared(MapEvent event) {
-      // Ignore it
-    }
-
-    @Override
-    public void mapEvicted(MapEvent event) {
-      // Ignore it
-    }
+  /**
+   * Only used for testing purpose
+   *
+   * @param logger
+   */
+  static void setLogger(Logger logger) {
+    AppStateClusterImpl.LOGGER = logger;
   }
+
 }
diff --git a/server/sonar-process-monitor/src/main/java/org/sonar/application/cluster/ClusterProperties.java b/server/sonar-process-monitor/src/main/java/org/sonar/application/cluster/ClusterProperties.java
index e8287c24d74..f33877f4562 100644
--- a/server/sonar-process-monitor/src/main/java/org/sonar/application/cluster/ClusterProperties.java
+++ b/server/sonar-process-monitor/src/main/java/org/sonar/application/cluster/ClusterProperties.java
@@ -82,6 +82,7 @@ public final class ClusterProperties {
     if (!enabled) {
       return;
     }
+
     // Test validity of port
     checkArgument(
       port > 0 && port < 65_536,
diff --git a/server/sonar-process-monitor/src/main/java/org/sonar/application/cluster/HazelcastCluster.java b/server/sonar-process-monitor/src/main/java/org/sonar/application/cluster/HazelcastCluster.java
new file mode 100644
index 00000000000..39b4b18b98f
--- /dev/null
+++ b/server/sonar-process-monitor/src/main/java/org/sonar/application/cluster/HazelcastCluster.java
@@ -0,0 +1,234 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package org.sonar.application.cluster;
+
+import com.hazelcast.config.Config;
+import com.hazelcast.config.JoinConfig;
+import com.hazelcast.config.NetworkConfig;
+import com.hazelcast.core.EntryEvent;
+import com.hazelcast.core.EntryListener;
+import com.hazelcast.core.Hazelcast;
+import com.hazelcast.core.HazelcastInstance;
+import com.hazelcast.core.IAtomicReference;
+import com.hazelcast.core.ILock;
+import com.hazelcast.core.MapEvent;
+import com.hazelcast.core.ReplicatedMap;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.sonar.application.AppStateListener;
+import org.sonar.process.ProcessId;
+
+import static java.util.stream.Collectors.toList;
+import static org.sonar.process.NetworkUtils.getHostName;
+
+public class HazelcastCluster implements AutoCloseable {
+  static final String OPERATIONAL_PROCESSES = "OPERATIONAL_PROCESSES";
+  static final String LEADER = "LEADER";
+  static final String HOSTNAME = "HOSTNAME";
+  static final String SONARQUBE_VERSION = "SONARQUBE_VERSION";
+
+  private final List<AppStateListener> listeners = new ArrayList<>();
+  private final ReplicatedMap<ClusterProcess, Boolean> operationalProcesses;
+  private final String operationalProcessListenerUUID;
+
+  final HazelcastInstance hzInstance;
+
+  private HazelcastCluster(Config hzConfig) {
+    // Create the Hazelcast instance
+    hzInstance = Hazelcast.newHazelcastInstance(hzConfig);
+
+    // Get or create the replicated map
+    operationalProcesses = hzInstance.getReplicatedMap(OPERATIONAL_PROCESSES);
+    operationalProcessListenerUUID = operationalProcesses.addEntryListener(new OperationalProcessListener());
+  }
+
+  String getLocalUuid() {
+    return hzInstance.getLocalEndpoint().getUuid();
+  }
+
+  String getName() {
+    return hzInstance.getConfig().getGroupConfig().getName();
+  }
+
+  List<String> getMembers() {
+    return hzInstance.getCluster().getMembers().stream()
+      .filter(m -> !m.localMember())
+      .map(m -> m.getStringAttribute(HOSTNAME))
+      .collect(toList());
+  }
+
+  void addListener(AppStateListener listener) {
+    listeners.add(listener);
+  }
+
+  boolean isOperational(ProcessId processId) {
+    for (Map.Entry<ClusterProcess, Boolean> entry : operationalProcesses.entrySet()) {
+      if (entry.getKey().getProcessId().equals(processId) && entry.getValue()) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  void setOperational(ProcessId processId) {
+    operationalProcesses.put(new ClusterProcess(getLocalUuid(), processId), Boolean.TRUE);
+  }
+
+  boolean tryToLockWebLeader() {
+    IAtomicReference<String> leader = hzInstance.getAtomicReference(LEADER);
+    if (leader.get() == null) {
+      ILock lock = hzInstance.getLock(LEADER);
+      lock.lock();
+      try {
+        if (leader.get() == null) {
+          leader.set(getLocalUuid());
+          return true;
+        } else {
+          return false;
+        }
+      } finally {
+        lock.unlock();
+      }
+    } else {
+      return false;
+    }
+  }
+
+  public void registerSonarQubeVersion(String sonarqubeVersion) {
+    IAtomicReference<String> sqVersion = hzInstance.getAtomicReference(SONARQUBE_VERSION);
+    if (sqVersion.get() == null) {
+      ILock lock = hzInstance.getLock(SONARQUBE_VERSION);
+      lock.lock();
+      try {
+        if (sqVersion.get() == null) {
+          sqVersion.set(sonarqubeVersion);
+        }
+      } finally {
+        lock.unlock();
+      }
+    }
+
+    String clusterVersion = sqVersion.get();
+    if (!sqVersion.get().equals(sonarqubeVersion)) {
+      throw new IllegalStateException(
+        String.format("The local version %s is not the same as the cluster %s", sonarqubeVersion, clusterVersion)
+      );
+    }
+  }
+
+  @Override
+  public void close() {
+    if (hzInstance != null) {
+      // Removing listeners
+      operationalProcesses.removeEntryListener(operationalProcessListenerUUID);
+
+      // Removing the operationalProcess from the replicated map
+      operationalProcesses.keySet().forEach(
+        clusterNodeProcess -> {
+          if (clusterNodeProcess.getNodeUuid().equals(getLocalUuid())) {
+            operationalProcesses.remove(clusterNodeProcess);
+          }
+        });
+
+      // Shutdown Hazelcast properly
+      hzInstance.shutdown();
+    }
+  }
+
+  public static HazelcastCluster create(ClusterProperties clusterProperties) {
+    Config hzConfig = new Config();
+    hzConfig.getGroupConfig().setName(clusterProperties.getName());
+
+    // Configure the network instance
+    NetworkConfig netConfig = hzConfig.getNetworkConfig();
+    netConfig
+      .setPort(clusterProperties.getPort())
+      .setReuseAddress(true);
+
+    if (!clusterProperties.getNetworkInterfaces().isEmpty()) {
+      netConfig.getInterfaces()
+        .setEnabled(true)
+        .setInterfaces(clusterProperties.getNetworkInterfaces());
+    }
+
+    // Only allowing TCP/IP configuration
+    JoinConfig joinConfig = netConfig.getJoin();
+    joinConfig.getAwsConfig().setEnabled(false);
+    joinConfig.getMulticastConfig().setEnabled(false);
+    joinConfig.getTcpIpConfig().setEnabled(true);
+    joinConfig.getTcpIpConfig().setMembers(clusterProperties.getHosts());
+
+    // Tweak HazelCast configuration
+    hzConfig
+      // Increase the number of tries
+      .setProperty("hazelcast.tcp.join.port.try.count", "10")
+      // Don't bind on all interfaces
+      .setProperty("hazelcast.socket.bind.any", "false")
+      // Don't phone home
+      .setProperty("hazelcast.phone.home.enabled", "false")
+      // Use slf4j for logging
+      .setProperty("hazelcast.logging.type", "slf4j");
+
+    // Trying to resolve the hostname
+    hzConfig.getMemberAttributeConfig().setStringAttribute(HOSTNAME, getHostName());
+
+    // We are not using the partition group of Hazelcast, so disabling it
+    hzConfig.getPartitionGroupConfig().setEnabled(false);
+    return new HazelcastCluster(hzConfig);
+  }
+
+  private class OperationalProcessListener implements EntryListener<ClusterProcess, Boolean> {
+    @Override
+    public void entryAdded(EntryEvent<ClusterProcess, Boolean> event) {
+      if (event.getValue()) {
+        listeners.forEach(appStateListener -> appStateListener.onAppStateOperational(event.getKey().getProcessId()));
+      }
+    }
+
+    @Override
+    public void entryRemoved(EntryEvent<ClusterProcess, Boolean> event) {
+      // Ignore it
+    }
+
+    @Override
+    public void entryUpdated(EntryEvent<ClusterProcess, Boolean> event) {
+      if (event.getValue()) {
+        listeners.forEach(appStateListener -> appStateListener.onAppStateOperational(event.getKey().getProcessId()));
+      }
+    }
+
+    @Override
+    public void entryEvicted(EntryEvent<ClusterProcess, Boolean> event) {
+      // Ignore it
+    }
+
+    @Override
+    public void mapCleared(MapEvent event) {
+      // Ignore it
+    }
+
+    @Override
+    public void mapEvicted(MapEvent event) {
+      // Ignore it
+    }
+  }
+}
diff --git a/server/sonar-process-monitor/src/test/java/org/sonar/application/cluster/AppStateClusterImplTest.java b/server/sonar-process-monitor/src/test/java/org/sonar/application/cluster/AppStateClusterImplTest.java
index 98d44cb0a4a..fe2bc98336e 100644
--- a/server/sonar-process-monitor/src/test/java/org/sonar/application/cluster/AppStateClusterImplTest.java
+++ b/server/sonar-process-monitor/src/test/java/org/sonar/application/cluster/AppStateClusterImplTest.java
@@ -20,28 +20,28 @@
 package org.sonar.application.cluster;
 
 import com.hazelcast.core.HazelcastInstance;
-import com.hazelcast.core.ReplicatedMap;
-import java.net.InetAddress;
-import java.util.UUID;
+import java.io.IOException;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.DisableOnDebug;
 import org.junit.rules.ExpectedException;
 import org.junit.rules.TestRule;
 import org.junit.rules.Timeout;
+import org.slf4j.Logger;
 import org.sonar.application.AppStateListener;
 import org.sonar.application.config.TestAppSettings;
 import org.sonar.process.ProcessId;
 import org.sonar.process.ProcessProperties;
 
-import static org.assertj.core.api.Java6Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.timeout;
 import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
-import static org.sonar.application.cluster.AppStateClusterImpl.OPERATIONAL_PROCESSES;
-import static org.sonar.application.cluster.AppStateClusterImpl.SONARQUBE_VERSION;
+import static org.sonar.application.cluster.HazelcastCluster.SONARQUBE_VERSION;
 import static org.sonar.application.cluster.HazelcastTestHelper.createHazelcastClient;
+import static org.sonar.application.cluster.HazelcastTestHelper.newClusterSettings;
 
 public class AppStateClusterImplTest {
 
@@ -73,6 +73,23 @@ public class AppStateClusterImplTest {
   }
 
   @Test
+  public void log_when_sonarqube_is_joining_a_cluster () throws IOException, InterruptedException, IllegalAccessException, NoSuchFieldException {
+    // Now launch an instance that try to be part of the hzInstance cluster
+    TestAppSettings settings = newClusterSettings();
+
+    Logger logger = mock(Logger.class);
+    AppStateClusterImpl.setLogger(logger);
+
+    try (AppStateClusterImpl appStateCluster = new AppStateClusterImpl(settings)) {
+      verify(logger).info(
+        eq("Joined the cluster [{}] that contains the following hosts : [{}]"),
+        eq("sonarqube"),
+        anyString()
+      );
+    }
+  }
+
+  @Test
   public void test_listeners() throws InterruptedException {
     AppStateListener listener = mock(AppStateListener.class);
     try (AppStateClusterImpl underTest = new AppStateClusterImpl(newClusterSettings())) {
@@ -89,57 +106,35 @@ public class AppStateClusterImplTest {
   }
 
   @Test
-  public void simulate_network_cluster() throws InterruptedException {
+  public void registerSonarQubeVersion_publishes_version_on_first_call() {
     TestAppSettings settings = newClusterSettings();
-    settings.set(ProcessProperties.CLUSTER_NETWORK_INTERFACES, InetAddress.getLoopbackAddress().getHostAddress());
-    AppStateListener listener = mock(AppStateListener.class);
 
     try (AppStateClusterImpl appStateCluster = new AppStateClusterImpl(settings)) {
-      appStateCluster.addListener(listener);
+      appStateCluster.registerSonarQubeVersion("6.4.1.5");
 
       HazelcastInstance hzInstance = createHazelcastClient(appStateCluster);
-      String uuid = UUID.randomUUID().toString();
-      ReplicatedMap<ClusterProcess, Boolean> replicatedMap = hzInstance.getReplicatedMap(OPERATIONAL_PROCESSES);
-      // process is not up yet --> no events are sent to listeners
-      replicatedMap.put(
-        new ClusterProcess(uuid, ProcessId.ELASTICSEARCH),
-        Boolean.FALSE);
-
-      // process is up yet --> notify listeners
-      replicatedMap.replace(
-        new ClusterProcess(uuid, ProcessId.ELASTICSEARCH),
-        Boolean.TRUE);
-
-      // should be called only once
-      verify(listener, timeout(20_000)).onAppStateOperational(ProcessId.ELASTICSEARCH);
-      verifyNoMoreInteractions(listener);
-
-      hzInstance.shutdown();
+      assertThat(hzInstance.getAtomicReference(SONARQUBE_VERSION).get())
+        .isNotNull()
+        .isInstanceOf(String.class)
+        .isEqualTo("6.4.1.5");
     }
   }
 
   @Test
-  public void registerSonarQubeVersion_publishes_version_on_first_call() {
+  public void reset_throws_always_ISE() {
     TestAppSettings settings = newClusterSettings();
 
     try (AppStateClusterImpl appStateCluster = new AppStateClusterImpl(settings)) {
-      appStateCluster.registerSonarQubeVersion("6.4.1.5");
-
-      HazelcastInstance hzInstance = createHazelcastClient(appStateCluster);
-      assertThat(hzInstance.getAtomicReference(SONARQUBE_VERSION).get())
-        .isNotNull()
-        .isInstanceOf(String.class)
-        .isEqualTo("6.4.1.5");
+      expectedException.expect(IllegalStateException.class);
+      expectedException.expectMessage("state reset is not supported in cluster mode");
+      appStateCluster.reset();
     }
   }
 
   @Test
   public void registerSonarQubeVersion_throws_ISE_if_initial_version_is_different() throws Exception {
     // Now launch an instance that try to be part of the hzInstance cluster
-    TestAppSettings settings = new TestAppSettings();
-    settings.set(ProcessProperties.CLUSTER_ENABLED, "true");
-    settings.set(ProcessProperties.CLUSTER_NAME, "sonarqube");
-
+    TestAppSettings settings = newClusterSettings();
 
     try (AppStateClusterImpl appStateCluster = new AppStateClusterImpl(settings)) {
       // Register first version
@@ -152,11 +147,4 @@ public class AppStateClusterImplTest {
       appStateCluster.registerSonarQubeVersion("2.0.0");
     }
   }
-
-  private static TestAppSettings newClusterSettings() {
-    TestAppSettings settings = new TestAppSettings();
-    settings.set(ProcessProperties.CLUSTER_ENABLED, "true");
-    settings.set(ProcessProperties.CLUSTER_NAME, "sonarqube");
-    return settings;
-  }
 }
diff --git a/server/sonar-process-monitor/src/test/java/org/sonar/application/cluster/HazelcastClusterTest.java b/server/sonar-process-monitor/src/test/java/org/sonar/application/cluster/HazelcastClusterTest.java
new file mode 100644
index 00000000000..a265ff1ea1d
--- /dev/null
+++ b/server/sonar-process-monitor/src/test/java/org/sonar/application/cluster/HazelcastClusterTest.java
@@ -0,0 +1,200 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.application.cluster;
+
+import com.hazelcast.core.HazelcastInstance;
+import com.hazelcast.core.ReplicatedMap;
+import java.net.InetAddress;
+import java.util.AbstractMap;
+import java.util.UUID;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.DisableOnDebug;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.TestRule;
+import org.junit.rules.Timeout;
+import org.sonar.application.AppStateListener;
+import org.sonar.application.config.TestAppSettings;
+import org.sonar.process.ProcessId;
+import org.sonar.process.ProcessProperties;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.sonar.application.cluster.HazelcastCluster.LEADER;
+import static org.sonar.application.cluster.HazelcastCluster.OPERATIONAL_PROCESSES;
+import static org.sonar.application.cluster.HazelcastCluster.SONARQUBE_VERSION;
+import static org.sonar.application.cluster.HazelcastTestHelper.createHazelcastClient;
+import static org.sonar.application.cluster.HazelcastTestHelper.newClusterSettings;
+import static org.sonar.process.ProcessProperties.CLUSTER_NAME;
+
+public class HazelcastClusterTest {
+  @Rule
+  public TestRule safeGuard = new DisableOnDebug(Timeout.seconds(10));
+
+  @Rule
+  public ExpectedException expectedException = ExpectedException.none();
+
+  @Test
+  public void test_two_tryToLockWebLeader_must_return_true() {
+    ClusterProperties clusterProperties = new ClusterProperties(newClusterSettings());
+    try (HazelcastCluster hzCluster = HazelcastCluster.create(clusterProperties)) {
+      assertThat(hzCluster.tryToLockWebLeader()).isEqualTo(true);
+      assertThat(hzCluster.tryToLockWebLeader()).isEqualTo(false);
+    }
+  }
+
+  @Test
+  public void when_another_process_locked_webleader_tryToLockWebLeader_must_return_false() {
+    ClusterProperties clusterProperties = new ClusterProperties(newClusterSettings());
+    try (HazelcastCluster hzCluster = HazelcastCluster.create(clusterProperties)) {
+      HazelcastInstance hzInstance = createHazelcastClient(hzCluster);
+      hzInstance.getAtomicReference(LEADER).set("aaaa");
+      assertThat(hzCluster.tryToLockWebLeader()).isEqualTo(false);
+    }
+  }
+
+  @Test
+  public void members_must_be_empty_when_there_is_no_other_node() {
+    ClusterProperties clusterProperties = new ClusterProperties(newClusterSettings());
+    try (HazelcastCluster hzCluster = HazelcastCluster.create(clusterProperties)) {
+      assertThat(hzCluster.getMembers()).isEmpty();
+    }
+  }
+
+  @Test
+  public void set_operational_is_writing_to_cluster() {
+    ClusterProperties clusterProperties = new ClusterProperties(newClusterSettings());
+    try (HazelcastCluster hzCluster = HazelcastCluster.create(clusterProperties)) {
+
+      hzCluster.setOperational(ProcessId.ELASTICSEARCH);
+
+      assertThat(hzCluster.isOperational(ProcessId.ELASTICSEARCH)).isTrue();
+      assertThat(hzCluster.isOperational(ProcessId.WEB_SERVER)).isFalse();
+      assertThat(hzCluster.isOperational(ProcessId.COMPUTE_ENGINE)).isFalse();
+
+      // Connect via Hazelcast client to test values
+      HazelcastInstance hzInstance = createHazelcastClient(hzCluster);
+      ReplicatedMap<ClusterProcess, Boolean> operationalProcesses = hzInstance.getReplicatedMap(OPERATIONAL_PROCESSES);
+      assertThat(operationalProcesses)
+        .containsExactly(new AbstractMap.SimpleEntry<>(new ClusterProcess(hzCluster.getLocalUuid(), ProcessId.ELASTICSEARCH), Boolean.TRUE));
+    }
+  }
+
+  @Test
+  public void cluster_name_comes_from_configuration() {
+    TestAppSettings testAppSettings = newClusterSettings();
+    testAppSettings.set(CLUSTER_NAME, "a_cluster_");
+    ClusterProperties clusterProperties = new ClusterProperties(testAppSettings);
+    try (HazelcastCluster hzCluster = HazelcastCluster.create(clusterProperties)) {
+      assertThat(hzCluster.getName()).isEqualTo("a_cluster_");
+    }
+  }
+
+  @Test
+  public void localUUID_must_not_be_empty() {
+    ClusterProperties clusterProperties = new ClusterProperties(newClusterSettings());
+    try (HazelcastCluster hzCluster = HazelcastCluster.create(clusterProperties)) {
+      assertThat(hzCluster.getLocalUuid()).isNotEmpty();
+    }
+  }
+
+  @Test
+  public void when_a_process_is_set_operational_listener_must_be_triggered() {
+    ClusterProperties clusterProperties = new ClusterProperties(newClusterSettings());
+    try (HazelcastCluster hzCluster = HazelcastCluster.create(clusterProperties)) {
+      AppStateListener listener = mock(AppStateListener.class);
+      hzCluster.addListener(listener);
+
+      // ElasticSearch is not operational
+      assertThat(hzCluster.isOperational(ProcessId.ELASTICSEARCH)).isFalse();
+
+      // Simulate a node that set ElasticSearch operational
+      HazelcastInstance hzInstance = createHazelcastClient(hzCluster);
+      ReplicatedMap<ClusterProcess, Boolean> operationalProcesses = hzInstance.getReplicatedMap(OPERATIONAL_PROCESSES);
+      operationalProcesses.put(new ClusterProcess(UUID.randomUUID().toString(), ProcessId.ELASTICSEARCH), Boolean.TRUE);
+      verify(listener, timeout(20_000)).onAppStateOperational(ProcessId.ELASTICSEARCH);
+      verifyNoMoreInteractions(listener);
+
+      // ElasticSearch is operational
+      assertThat(hzCluster.isOperational(ProcessId.ELASTICSEARCH)).isTrue();
+    }
+  }
+
+
+  @Test
+  public void registerSonarQubeVersion_publishes_version_on_first_call() {
+    ClusterProperties clusterProperties = new ClusterProperties(newClusterSettings());
+    try (HazelcastCluster hzCluster = HazelcastCluster.create(clusterProperties)) {
+      hzCluster.registerSonarQubeVersion("1.0.0.0");
+
+      HazelcastInstance hzInstance = createHazelcastClient(hzCluster);
+      assertThat(hzInstance.getAtomicReference(SONARQUBE_VERSION).get()).isEqualTo("1.0.0.0");
+    }
+  }
+
+
+  @Test
+  public void registerSonarQubeVersion_throws_ISE_if_initial_version_is_different() throws Exception {
+    ClusterProperties clusterProperties = new ClusterProperties(newClusterSettings());
+    try (HazelcastCluster hzCluster = HazelcastCluster.create(clusterProperties)) {
+      // Register first version
+      hzCluster.registerSonarQubeVersion("1.0.0");
+
+      expectedException.expect(IllegalStateException.class);
+      expectedException.expectMessage("The local version 2.0.0 is not the same as the cluster 1.0.0");
+
+      // Registering a second different version must trigger an exception
+      hzCluster.registerSonarQubeVersion("2.0.0");
+    }
+  }
+
+  @Test
+  public void simulate_network_cluster() throws InterruptedException {
+    TestAppSettings settings = newClusterSettings();
+    settings.set(ProcessProperties.CLUSTER_NETWORK_INTERFACES, InetAddress.getLoopbackAddress().getHostAddress());
+    AppStateListener listener = mock(AppStateListener.class);
+
+    try (AppStateClusterImpl appStateCluster = new AppStateClusterImpl(settings)) {
+      appStateCluster.addListener(listener);
+
+      HazelcastInstance hzInstance = createHazelcastClient(appStateCluster.getHazelcastCluster());
+      String uuid = UUID.randomUUID().toString();
+      ReplicatedMap<ClusterProcess, Boolean> replicatedMap = hzInstance.getReplicatedMap(OPERATIONAL_PROCESSES);
+      // process is not up yet --> no events are sent to listeners
+      replicatedMap.put(
+        new ClusterProcess(uuid, ProcessId.ELASTICSEARCH),
+        Boolean.FALSE);
+
+      // process is up yet --> notify listeners
+      replicatedMap.replace(
+        new ClusterProcess(uuid, ProcessId.ELASTICSEARCH),
+        Boolean.TRUE);
+
+      // should be called only once
+      verify(listener, timeout(20_000)).onAppStateOperational(ProcessId.ELASTICSEARCH);
+      verifyNoMoreInteractions(listener);
+
+      hzInstance.shutdown();
+    }
+  }
+}
diff --git a/server/sonar-process-monitor/src/test/java/org/sonar/application/cluster/HazelcastTestHelper.java b/server/sonar-process-monitor/src/test/java/org/sonar/application/cluster/HazelcastTestHelper.java
index 106390dc2d1..81033eb34f5 100644
--- a/server/sonar-process-monitor/src/test/java/org/sonar/application/cluster/HazelcastTestHelper.java
+++ b/server/sonar-process-monitor/src/test/java/org/sonar/application/cluster/HazelcastTestHelper.java
@@ -24,19 +24,34 @@ import com.hazelcast.client.HazelcastClient;
 import com.hazelcast.client.config.ClientConfig;
 import com.hazelcast.core.HazelcastInstance;
 import java.net.InetSocketAddress;
+import org.sonar.application.config.TestAppSettings;
+import org.sonar.process.ProcessProperties;
 
 public class HazelcastTestHelper {
 
-  static HazelcastInstance createHazelcastClient(AppStateClusterImpl appStateCluster) {
+  static HazelcastInstance createHazelcastClient(HazelcastCluster hzCluster) {
     ClientConfig clientConfig = new ClientConfig();
-    InetSocketAddress socketAddress = (InetSocketAddress) appStateCluster.hzInstance.getLocalEndpoint().getSocketAddress();
+    InetSocketAddress socketAddress = (InetSocketAddress) hzCluster.hzInstance.getLocalEndpoint().getSocketAddress();
 
     clientConfig.getNetworkConfig().getAddresses().add(
       String.format("%s:%d",
         socketAddress.getHostString(),
         socketAddress.getPort()
       ));
-    clientConfig.getGroupConfig().setName(appStateCluster.hzInstance.getConfig().getGroupConfig().getName());
+    clientConfig.getGroupConfig().setName(hzCluster.getName());
     return HazelcastClient.newHazelcastClient(clientConfig);
   }
+
+  static HazelcastInstance createHazelcastClient(AppStateClusterImpl appStateCluster) {
+    return createHazelcastClient(appStateCluster.getHazelcastCluster());
+  }
+
+
+  static TestAppSettings newClusterSettings() {
+    TestAppSettings settings = new TestAppSettings();
+    settings.set(ProcessProperties.CLUSTER_ENABLED, "true");
+    settings.set(ProcessProperties.CLUSTER_NAME, "sonarqube");
+    return settings;
+  }
+
 }