SONAR-8935 Add a log when joining a cluster

4 files changed, 259 insertions, 161 deletions

diff --git a/server/sonar-process-monitor/src/main/java/org/sonar/application/SchedulerImpl.java b/server/sonar-process-monitor/src/main/java/org/sonar/application/SchedulerImpl.java
index 4cce5a17467..8359d66e08d 100644
--- a/server/sonar-process-monitor/src/main/java/org/sonar/application/SchedulerImpl.java
+++ b/server/sonar-process-monitor/src/main/java/org/sonar/application/SchedulerImpl.java
@@ -161,7 +161,6 @@ public class SchedulerImpl implements Scheduler, ProcessEventListener, ProcessLi
     if (process != null) {
       process.stop(1, TimeUnit.MINUTES);
     }
-
   }
 
   /**
diff --git a/server/sonar-process-monitor/src/main/java/org/sonar/application/cluster/AppStateClusterImpl.java b/server/sonar-process-monitor/src/main/java/org/sonar/application/cluster/AppStateClusterImpl.java
index 3f25a987408..cae4d880572 100644
--- a/server/sonar-process-monitor/src/main/java/org/sonar/application/cluster/AppStateClusterImpl.java
+++ b/server/sonar-process-monitor/src/main/java/org/sonar/application/cluster/AppStateClusterImpl.java
@@ -20,41 +20,22 @@
 
 package org.sonar.application.cluster;
 
-import com.hazelcast.config.Config;
-import com.hazelcast.config.JoinConfig;
-import com.hazelcast.config.NetworkConfig;
-import com.hazelcast.core.EntryEvent;
-import com.hazelcast.core.EntryListener;
-import com.hazelcast.core.Hazelcast;
-import com.hazelcast.core.HazelcastInstance;
-import com.hazelcast.core.IAtomicReference;
-import com.hazelcast.core.ILock;
-import com.hazelcast.core.MapEvent;
-import com.hazelcast.core.ReplicatedMap;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
 import java.util.EnumMap;
-import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 import javax.annotation.Nonnull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.sonar.application.AppState;
 import org.sonar.application.AppStateListener;
 import org.sonar.application.config.AppSettings;
 import org.sonar.process.ProcessId;
 
 public class AppStateClusterImpl implements AppState {
-  static final String OPERATIONAL_PROCESSES = "OPERATIONAL_PROCESSES";
-  static final String LEADER = "LEADER";
+  private static Logger LOGGER = LoggerFactory.getLogger(AppStateClusterImpl.class);
 
-  static final String SONARQUBE_VERSION = "SONARQUBE_VERSION";
-
-  private final List<AppStateListener> listeners = new ArrayList<>();
-  private final ReplicatedMap<ClusterProcess, Boolean> operationalProcesses;
   private final Map<ProcessId, Boolean> localProcesses = new EnumMap<>(ProcessId.class);
-  private final String listenerUuid;
-
-  final HazelcastInstance hzInstance;
+  private final HazelcastCluster hazelcastCluster;
 
   public AppStateClusterImpl(AppSettings appSettings) {
     ClusterProperties clusterProperties = new ClusterProperties(appSettings);
@@ -64,56 +45,15 @@ public class AppStateClusterImpl implements AppState {
       throw new IllegalStateException("Cluster is not enabled on this instance");
     }
 
-    Config hzConfig = new Config();
-    try {
-      hzConfig.setInstanceName(InetAddress.getLocalHost().getHostName());
-    } catch (UnknownHostException e) {
-      // Ignore it
-    }
-
-    hzConfig.getGroupConfig().setName(clusterProperties.getName());
-
-    // Configure the network instance
-    NetworkConfig netConfig = hzConfig.getNetworkConfig();
-    netConfig
-      .setPort(clusterProperties.getPort())
-      .setReuseAddress(true);
-
-    if (!clusterProperties.getNetworkInterfaces().isEmpty()) {
-      netConfig.getInterfaces()
-        .setEnabled(true)
-        .setInterfaces(clusterProperties.getNetworkInterfaces());
-    }
+    hazelcastCluster = HazelcastCluster.create(clusterProperties);
 
-    // Only allowing TCP/IP configuration
-    JoinConfig joinConfig = netConfig.getJoin();
-    joinConfig.getAwsConfig().setEnabled(false);
-    joinConfig.getMulticastConfig().setEnabled(false);
-    joinConfig.getTcpIpConfig().setEnabled(true);
-    joinConfig.getTcpIpConfig().setMembers(clusterProperties.getHosts());
-
-    // Tweak HazelCast configuration
-    hzConfig
-      // Increase the number of tries
-      .setProperty("hazelcast.tcp.join.port.try.count", "10")
-      // Don't bind on all interfaces
-      .setProperty("hazelcast.socket.bind.any", "false")
-      // Don't phone home
-      .setProperty("hazelcast.phone.home.enabled", "false")
-      // Use slf4j for logging
-      .setProperty("hazelcast.logging.type", "slf4j");
-
-    // We are not using the partition group of Hazelcast, so disabling it
-    hzConfig.getPartitionGroupConfig().setEnabled(false);
-
-    hzInstance = Hazelcast.newHazelcastInstance(hzConfig);
-    operationalProcesses = hzInstance.getReplicatedMap(OPERATIONAL_PROCESSES);
-    listenerUuid = operationalProcesses.addEntryListener(new OperationalProcessListener());
+    String members = hazelcastCluster.getMembers().stream().collect(Collectors.joining(","));
+    LOGGER.info("Joined the cluster [{}] that contains the following hosts : [{}]", hazelcastCluster.getName(), members);
   }
 
   @Override
   public void addListener(@Nonnull AppStateListener listener) {
-    listeners.add(listener);
+    hazelcastCluster.addListener(listener);
   }
 
   @Override
@@ -121,39 +61,18 @@ public class AppStateClusterImpl implements AppState {
     if (local) {
       return localProcesses.computeIfAbsent(processId, p -> false);
     }
-    for (Map.Entry<ClusterProcess, Boolean> entry : operationalProcesses.entrySet()) {
-      if (entry.getKey().getProcessId().equals(processId) && entry.getValue()) {
-        return true;
-      }
-    }
-    return false;
+    return hazelcastCluster.isOperational(processId);
   }
 
   @Override
   public void setOperational(@Nonnull ProcessId processId) {
     localProcesses.put(processId, true);
-    operationalProcesses.put(new ClusterProcess(getLocalUuid(), processId), Boolean.TRUE);
+    hazelcastCluster.setOperational(processId);
   }
 
   @Override
   public boolean tryToLockWebLeader() {
-    IAtomicReference<String> leader = hzInstance.getAtomicReference(LEADER);
-    if (leader.get() == null) {
-      ILock lock = hzInstance.getLock(LEADER);
-      lock.lock();
-      try {
-        if (leader.get() == null) {
-          leader.set(getLocalUuid());
-          return true;
-        } else {
-          return false;
-        }
-      } finally {
-        lock.unlock();
-      }
-    } else {
-      return false;
-    }
+    return hazelcastCluster.tryToLockWebLeader();
   }
 
   @Override
@@ -163,80 +82,25 @@ public class AppStateClusterImpl implements AppState {
 
   @Override
   public void close() {
-    if (hzInstance != null) {
-      operationalProcesses.removeEntryListener(listenerUuid);
-      operationalProcesses.keySet().forEach(
-        clusterNodeProcess -> {
-          if (clusterNodeProcess.getNodeUuid().equals(getLocalUuid())) {
-            operationalProcesses.remove(clusterNodeProcess);
-          }
-        });
-      hzInstance.shutdown();
-    }
+    hazelcastCluster.close();
   }
 
   @Override
   public void registerSonarQubeVersion(String sonarqubeVersion) {
-    IAtomicReference<String> sqVersion = hzInstance.getAtomicReference(SONARQUBE_VERSION);
-    if (sqVersion.get() == null) {
-      ILock lock = hzInstance.getLock(SONARQUBE_VERSION);
-      lock.lock();
-      try {
-        if (sqVersion.get() == null) {
-          sqVersion.set(sonarqubeVersion);
-        }
-      } finally {
-        lock.unlock();
-      }
-    }
-
-    String clusterVersion = sqVersion.get();
-    if (!sqVersion.get().equals(sonarqubeVersion)) {
-      hzInstance.shutdown();
-      throw new IllegalStateException(
-        String.format("The local version %s is not the same as the cluster %s", sonarqubeVersion, clusterVersion)
-      );
-    }
+    hazelcastCluster.registerSonarQubeVersion(sonarqubeVersion);
   }
 
-  private String getLocalUuid() {
-    return hzInstance.getLocalEndpoint().getUuid();
+  HazelcastCluster getHazelcastCluster() {
+    return hazelcastCluster;
   }
 
-  private class OperationalProcessListener implements EntryListener<ClusterProcess, Boolean> {
-
-    @Override
-    public void entryAdded(EntryEvent<ClusterProcess, Boolean> event) {
-      if (event.getValue()) {
-        listeners.forEach(appStateListener -> appStateListener.onAppStateOperational(event.getKey().getProcessId()));
-      }
-    }
-
-    @Override
-    public void entryRemoved(EntryEvent<ClusterProcess, Boolean> event) {
-      // Ignore it
-    }
-
-    @Override
-    public void entryUpdated(EntryEvent<ClusterProcess, Boolean> event) {
-      if (event.getValue()) {
-        listeners.forEach(appStateListener -> appStateListener.onAppStateOperational(event.getKey().getProcessId()));
-      }
-    }
-
-    @Override
-    public void entryEvicted(EntryEvent<ClusterProcess, Boolean> event) {
-      // Ignore it
-    }
-
-    @Override
-    public void mapCleared(MapEvent event) {
-      // Ignore it
-    }
-
-    @Override
-    public void mapEvicted(MapEvent event) {
-      // Ignore it
-    }
+  /**
+   * Only used for testing purpose
+   *
+   * @param logger
+   */
+  static void setLogger(Logger logger) {
+    AppStateClusterImpl.LOGGER = logger;
   }
+
 }
diff --git a/server/sonar-process-monitor/src/main/java/org/sonar/application/cluster/ClusterProperties.java b/server/sonar-process-monitor/src/main/java/org/sonar/application/cluster/ClusterProperties.java
index e8287c24d74..f33877f4562 100644
--- a/server/sonar-process-monitor/src/main/java/org/sonar/application/cluster/ClusterProperties.java
+++ b/server/sonar-process-monitor/src/main/java/org/sonar/application/cluster/ClusterProperties.java
@@ -82,6 +82,7 @@ public final class ClusterProperties {
     if (!enabled) {
       return;
     }
+
     // Test validity of port
     checkArgument(
       port > 0 && port < 65_536,
diff --git a/server/sonar-process-monitor/src/main/java/org/sonar/application/cluster/HazelcastCluster.java b/server/sonar-process-monitor/src/main/java/org/sonar/application/cluster/HazelcastCluster.java
new file mode 100644
index 00000000000..39b4b18b98f
--- /dev/null
+++ b/server/sonar-process-monitor/src/main/java/org/sonar/application/cluster/HazelcastCluster.java
@@ -0,0 +1,234 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package org.sonar.application.cluster;
+
+import com.hazelcast.config.Config;
+import com.hazelcast.config.JoinConfig;
+import com.hazelcast.config.NetworkConfig;
+import com.hazelcast.core.EntryEvent;
+import com.hazelcast.core.EntryListener;
+import com.hazelcast.core.Hazelcast;
+import com.hazelcast.core.HazelcastInstance;
+import com.hazelcast.core.IAtomicReference;
+import com.hazelcast.core.ILock;
+import com.hazelcast.core.MapEvent;
+import com.hazelcast.core.ReplicatedMap;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.sonar.application.AppStateListener;
+import org.sonar.process.ProcessId;
+
+import static java.util.stream.Collectors.toList;
+import static org.sonar.process.NetworkUtils.getHostName;
+
+public class HazelcastCluster implements AutoCloseable {
+  static final String OPERATIONAL_PROCESSES = "OPERATIONAL_PROCESSES";
+  static final String LEADER = "LEADER";
+  static final String HOSTNAME = "HOSTNAME";
+  static final String SONARQUBE_VERSION = "SONARQUBE_VERSION";
+
+  private final List<AppStateListener> listeners = new ArrayList<>();
+  private final ReplicatedMap<ClusterProcess, Boolean> operationalProcesses;
+  private final String operationalProcessListenerUUID;
+
+  final HazelcastInstance hzInstance;
+
+  private HazelcastCluster(Config hzConfig) {
+    // Create the Hazelcast instance
+    hzInstance = Hazelcast.newHazelcastInstance(hzConfig);
+
+    // Get or create the replicated map
+    operationalProcesses = hzInstance.getReplicatedMap(OPERATIONAL_PROCESSES);
+    operationalProcessListenerUUID = operationalProcesses.addEntryListener(new OperationalProcessListener());
+  }
+
+  String getLocalUuid() {
+    return hzInstance.getLocalEndpoint().getUuid();
+  }
+
+  String getName() {
+    return hzInstance.getConfig().getGroupConfig().getName();
+  }
+
+  List<String> getMembers() {
+    return hzInstance.getCluster().getMembers().stream()
+      .filter(m -> !m.localMember())
+      .map(m -> m.getStringAttribute(HOSTNAME))
+      .collect(toList());
+  }
+
+  void addListener(AppStateListener listener) {
+    listeners.add(listener);
+  }
+
+  boolean isOperational(ProcessId processId) {
+    for (Map.Entry<ClusterProcess, Boolean> entry : operationalProcesses.entrySet()) {
+      if (entry.getKey().getProcessId().equals(processId) && entry.getValue()) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  void setOperational(ProcessId processId) {
+    operationalProcesses.put(new ClusterProcess(getLocalUuid(), processId), Boolean.TRUE);
+  }
+
+  boolean tryToLockWebLeader() {
+    IAtomicReference<String> leader = hzInstance.getAtomicReference(LEADER);
+    if (leader.get() == null) {
+      ILock lock = hzInstance.getLock(LEADER);
+      lock.lock();
+      try {
+        if (leader.get() == null) {
+          leader.set(getLocalUuid());
+          return true;
+        } else {
+          return false;
+        }
+      } finally {
+        lock.unlock();
+      }
+    } else {
+      return false;
+    }
+  }
+
+  public void registerSonarQubeVersion(String sonarqubeVersion) {
+    IAtomicReference<String> sqVersion = hzInstance.getAtomicReference(SONARQUBE_VERSION);
+    if (sqVersion.get() == null) {
+      ILock lock = hzInstance.getLock(SONARQUBE_VERSION);
+      lock.lock();
+      try {
+        if (sqVersion.get() == null) {
+          sqVersion.set(sonarqubeVersion);
+        }
+      } finally {
+        lock.unlock();
+      }
+    }
+
+    String clusterVersion = sqVersion.get();
+    if (!sqVersion.get().equals(sonarqubeVersion)) {
+      throw new IllegalStateException(
+        String.format("The local version %s is not the same as the cluster %s", sonarqubeVersion, clusterVersion)
+      );
+    }
+  }
+
+  @Override
+  public void close() {
+    if (hzInstance != null) {
+      // Removing listeners
+      operationalProcesses.removeEntryListener(operationalProcessListenerUUID);
+
+      // Removing the operationalProcess from the replicated map
+      operationalProcesses.keySet().forEach(
+        clusterNodeProcess -> {
+          if (clusterNodeProcess.getNodeUuid().equals(getLocalUuid())) {
+            operationalProcesses.remove(clusterNodeProcess);
+          }
+        });
+
+      // Shutdown Hazelcast properly
+      hzInstance.shutdown();
+    }
+  }
+
+  public static HazelcastCluster create(ClusterProperties clusterProperties) {
+    Config hzConfig = new Config();
+    hzConfig.getGroupConfig().setName(clusterProperties.getName());
+
+    // Configure the network instance
+    NetworkConfig netConfig = hzConfig.getNetworkConfig();
+    netConfig
+      .setPort(clusterProperties.getPort())
+      .setReuseAddress(true);
+
+    if (!clusterProperties.getNetworkInterfaces().isEmpty()) {
+      netConfig.getInterfaces()
+        .setEnabled(true)
+        .setInterfaces(clusterProperties.getNetworkInterfaces());
+    }
+
+    // Only allowing TCP/IP configuration
+    JoinConfig joinConfig = netConfig.getJoin();
+    joinConfig.getAwsConfig().setEnabled(false);
+    joinConfig.getMulticastConfig().setEnabled(false);
+    joinConfig.getTcpIpConfig().setEnabled(true);
+    joinConfig.getTcpIpConfig().setMembers(clusterProperties.getHosts());
+
+    // Tweak HazelCast configuration
+    hzConfig
+      // Increase the number of tries
+      .setProperty("hazelcast.tcp.join.port.try.count", "10")
+      // Don't bind on all interfaces
+      .setProperty("hazelcast.socket.bind.any", "false")
+      // Don't phone home
+      .setProperty("hazelcast.phone.home.enabled", "false")
+      // Use slf4j for logging
+      .setProperty("hazelcast.logging.type", "slf4j");
+
+    // Trying to resolve the hostname
+    hzConfig.getMemberAttributeConfig().setStringAttribute(HOSTNAME, getHostName());
+
+    // We are not using the partition group of Hazelcast, so disabling it
+    hzConfig.getPartitionGroupConfig().setEnabled(false);
+    return new HazelcastCluster(hzConfig);
+  }
+
+  private class OperationalProcessListener implements EntryListener<ClusterProcess, Boolean> {
+    @Override
+    public void entryAdded(EntryEvent<ClusterProcess, Boolean> event) {
+      if (event.getValue()) {
+        listeners.forEach(appStateListener -> appStateListener.onAppStateOperational(event.getKey().getProcessId()));
+      }
+    }
+
+    @Override
+    public void entryRemoved(EntryEvent<ClusterProcess, Boolean> event) {
+      // Ignore it
+    }
+
+    @Override
+    public void entryUpdated(EntryEvent<ClusterProcess, Boolean> event) {
+      if (event.getValue()) {
+        listeners.forEach(appStateListener -> appStateListener.onAppStateOperational(event.getKey().getProcessId()));
+      }
+    }
+
+    @Override
+    public void entryEvicted(EntryEvent<ClusterProcess, Boolean> event) {
+      // Ignore it
+    }
+
+    @Override
+    public void mapCleared(MapEvent event) {
+      // Ignore it
+    }
+
+    @Override
+    public void mapEvicted(MapEvent event) {
+      // Ignore it
+    }
+  }
+}