SONAR-9741 use cluster time to timestamp NodeHealth info shared in HZ

3 files changed, 111 insertions, 17 deletions

diff --git a/server/sonar-cluster/src/main/java/org/sonar/cluster/health/SharedHealthStateImpl.java b/server/sonar-cluster/src/main/java/org/sonar/cluster/health/SharedHealthStateImpl.java
index 99a48bf2548..2957fc297e7 100644
--- a/server/sonar-cluster/src/main/java/org/sonar/cluster/health/SharedHealthStateImpl.java
+++ b/server/sonar-cluster/src/main/java/org/sonar/cluster/health/SharedHealthStateImpl.java
@@ -44,16 +44,16 @@ public class SharedHealthStateImpl implements SharedHealthState {
   public void writeMine(NodeHealth nodeHealth) {
     requireNonNull(nodeHealth, "nodeHealth can't be null");
 
-    Map<String, NodeHealth> sqHealthState = hazelcastClient.getReplicatedMap(SQ_HEALTH_STATE_REPLICATED_MAP_IDENTIFIER);
+    Map<String, TimestampedNodeHealth> sqHealthState = readReplicatedMap();
     if (LOG.isTraceEnabled()) {
       LOG.trace("Reading {} and adding {}", new HashMap<>(sqHealthState), nodeHealth);
     }
-    sqHealthState.put(hazelcastClient.getUUID(), nodeHealth);
+    sqHealthState.put(hazelcastClient.getUUID(), new TimestampedNodeHealth(nodeHealth, hazelcastClient.getClusterTime()));
   }
 
   @Override
   public void clearMine() {
-    Map<String, NodeHealth> sqHealthState = hazelcastClient.getReplicatedMap(SQ_HEALTH_STATE_REPLICATED_MAP_IDENTIFIER);
+    Map<String, TimestampedNodeHealth> sqHealthState = readReplicatedMap();
     String clientUUID = hazelcastClient.getUUID();
     if (LOG.isTraceEnabled()) {
       LOG.trace("Reading {} and clearing for {}", new HashMap<>(sqHealthState), clientUUID);
@@ -63,14 +63,20 @@ public class SharedHealthStateImpl implements SharedHealthState {
 
   @Override
   public Set<NodeHealth> readAll() {
-    Map<String, NodeHealth> sqHealthState = hazelcastClient.getReplicatedMap(SQ_HEALTH_STATE_REPLICATED_MAP_IDENTIFIER);
+    Map<String, TimestampedNodeHealth> sqHealthState = readReplicatedMap();
     Set<String> hzMemberUUIDs = hazelcastClient.getMemberUuids();
     Set<NodeHealth> existingNodeHealths = sqHealthState.entrySet().stream()
-      .filter(entry -> hzMemberUUIDs.contains(entry.getKey())).map(Map.Entry::getValue)
+      .filter(entry -> hzMemberUUIDs.contains(entry.getKey()))
+      .map(entry -> entry.getValue().getNodeHealth())
       .collect(Collectors.toSet());
     if (LOG.isTraceEnabled()) {
       LOG.trace("Reading {} and keeping {}", new HashMap<>(sqHealthState), existingNodeHealths);
     }
     return ImmutableSet.copyOf(existingNodeHealths);
   }
+
+  private Map<String, TimestampedNodeHealth> readReplicatedMap() {
+    return hazelcastClient.getReplicatedMap(SQ_HEALTH_STATE_REPLICATED_MAP_IDENTIFIER);
+  }
+
 }
diff --git a/server/sonar-cluster/src/main/java/org/sonar/cluster/health/TimestampedNodeHealth.java b/server/sonar-cluster/src/main/java/org/sonar/cluster/health/TimestampedNodeHealth.java
new file mode 100644
index 00000000000..5f6f9c0579f
--- /dev/null
+++ b/server/sonar-cluster/src/main/java/org/sonar/cluster/health/TimestampedNodeHealth.java
@@ -0,0 +1,83 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.cluster.health;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Objects;
+
+public final class TimestampedNodeHealth implements Externalizable {
+  private NodeHealth nodeHealth;
+  private long timestamp;
+
+  public TimestampedNodeHealth() {
+    // required by Externalizable
+  }
+
+  public TimestampedNodeHealth(NodeHealth nodeHealth, long timestamp) {
+    this.nodeHealth = nodeHealth;
+    this.timestamp = timestamp;
+  }
+
+  public NodeHealth getNodeHealth() {
+    return nodeHealth;
+  }
+
+  public long getTimestamp() {
+    return timestamp;
+  }
+
+  @Override
+  public void writeExternal(ObjectOutput out) throws IOException {
+    out.writeLong(timestamp);
+    out.writeObject(nodeHealth);
+  }
+
+  @Override
+  public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+    this.timestamp = in.readLong();
+    this.nodeHealth = (NodeHealth) in.readObject();
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    TimestampedNodeHealth that = (TimestampedNodeHealth) o;
+    return timestamp == that.timestamp &&
+        Objects.equals(nodeHealth, that.nodeHealth);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(nodeHealth, timestamp);
+  }
+
+  @Override
+  public String toString() {
+    return nodeHealth + "@" + timestamp;
+  }
+}
diff --git a/server/sonar-cluster/src/test/java/org/sonar/cluster/health/SharedHealthStateImplTest.java b/server/sonar-cluster/src/test/java/org/sonar/cluster/health/SharedHealthStateImplTest.java
index 5b74e5579dc..bef109c26b5 100644
--- a/server/sonar-cluster/src/test/java/org/sonar/cluster/health/SharedHealthStateImplTest.java
+++ b/server/sonar-cluster/src/test/java/org/sonar/cluster/health/SharedHealthStateImplTest.java
@@ -64,15 +64,17 @@ public class SharedHealthStateImplTest {
   @Test
   public void write_put_arg_into_map_sq_health_state_under_current_client_uuid() {
     NodeHealth nodeHealth = randomNodeHealth();
-    Map<String, NodeHealth> map = new HashMap<>();
+    Map<String, TimestampedNodeHealth> map = new HashMap<>();
     doReturn(map).when(hazelcastClient).getReplicatedMap(MAP_SQ_HEALTH_STATE);
+    long clusterTime = random.nextLong();
     String uuid = randomAlphanumeric(5);
     when(hazelcastClient.getUUID()).thenReturn(uuid);
+    when(hazelcastClient.getClusterTime()).thenReturn(clusterTime);
 
     underTest.writeMine(nodeHealth);
 
     assertThat(map.size()).isEqualTo(1);
-    assertThat(map.get(uuid)).isSameAs(nodeHealth);
+    assertThat(map.get(uuid)).isEqualTo(new TimestampedNodeHealth(nodeHealth, clusterTime));
     assertThat(logTester.logs()).isEmpty();
   }
 
@@ -80,8 +82,8 @@ public class SharedHealthStateImplTest {
   public void write_logs_map_sq_health_state_content_and_NodeHealth_to_be_added_if_TRACE() {
     logTester.setLevel(LoggerLevel.TRACE);
     NodeHealth newNodeHealth = randomNodeHealth();
-    Map<String, NodeHealth> map = new HashMap<>();
-    map.put(randomAlphanumeric(4), randomNodeHealth());
+    Map<String, TimestampedNodeHealth> map = new HashMap<>();
+    map.put(randomAlphanumeric(4), new TimestampedNodeHealth(randomNodeHealth(), random.nextLong()));
     doReturn(new HashMap<>(map)).when(hazelcastClient).getReplicatedMap(MAP_SQ_HEALTH_STATE);
     String uuid = randomAlphanumeric(5);
     when(hazelcastClient.getUUID()).thenReturn(uuid);
@@ -96,12 +98,13 @@ public class SharedHealthStateImplTest {
   @Test
   public void readAll_returns_all_NodeHealth_in_map_sq_health_state_for_existing_client_uuids() {
     NodeHealth[] nodeHealths = IntStream.range(0, 1 + random.nextInt(6)).mapToObj(i -> randomNodeHealth()).toArray(NodeHealth[]::new);
-    Map<String, NodeHealth> allNodeHealths = new HashMap<>();
+    Map<String, TimestampedNodeHealth> allNodeHealths = new HashMap<>();
     Map<String, NodeHealth> expected = new HashMap<>();
     String randomUuidBase = randomAlphanumeric(5);
     for (int i = 0; i < nodeHealths.length; i++) {
       String memberUuid = randomUuidBase + i;
-      allNodeHealths.put(memberUuid, nodeHealths[i]);
+      TimestampedNodeHealth timestampedNodeHealth = new TimestampedNodeHealth(nodeHealths[i], random.nextLong());
+      allNodeHealths.put(memberUuid, timestampedNodeHealth);
       if (random.nextBoolean()) {
         expected.put(memberUuid, nodeHealths[i]);
       }
@@ -109,15 +112,17 @@ public class SharedHealthStateImplTest {
     doReturn(allNodeHealths).when(hazelcastClient).getReplicatedMap(MAP_SQ_HEALTH_STATE);
     when(hazelcastClient.getMemberUuids()).thenReturn(expected.keySet());
 
-    assertThat(underTest.readAll()).containsOnly(expected.values().stream().toArray(NodeHealth[]::new));
+    assertThat(underTest.readAll())
+      .containsOnly(expected.values().stream().toArray(NodeHealth[]::new));
     assertThat(logTester.logs()).isEmpty();
   }
 
   @Test
   public void readAll_logs_map_sq_health_state_content_and_the_content_effectively_returned_if_TRACE() {
     logTester.setLevel(LoggerLevel.TRACE);
-    Map<String, NodeHealth> map = new HashMap<>();
-    map.put(randomAlphanumeric(44), randomNodeHealth());
+    Map<String, TimestampedNodeHealth> map = new HashMap<>();
+    map.put(randomAlphanumeric(44), new TimestampedNodeHealth(randomNodeHealth(), random.nextLong()));
+    when(hazelcastClient.getClusterTime()).thenReturn(random.nextLong());
     doReturn(map).when(hazelcastClient).getReplicatedMap(MAP_SQ_HEALTH_STATE);
 
     underTest.readAll();
@@ -128,7 +133,7 @@ public class SharedHealthStateImplTest {
 
   @Test
   public void clearMine_clears_entry_into_map_sq_health_state_under_current_client_uuid() {
-    Map<String, NodeHealth> map = mock(Map.class);
+    Map<String, TimestampedNodeHealth> map = mock(Map.class);
     doReturn(map).when(hazelcastClient).getReplicatedMap(MAP_SQ_HEALTH_STATE);
     String uuid = randomAlphanumeric(5);
     when(hazelcastClient.getUUID()).thenReturn(uuid);
@@ -143,8 +148,8 @@ public class SharedHealthStateImplTest {
   @Test
   public void clearMine_logs_map_sq_health_state_and_current_client_uuid_if_TRACE() {
     logTester.setLevel(LoggerLevel.TRACE);
-    Map<String, NodeHealth> map = new HashMap<>();
-    map.put(randomAlphanumeric(4), randomNodeHealth());
+    Map<String, TimestampedNodeHealth> map = new HashMap<>();
+    map.put(randomAlphanumeric(4), new TimestampedNodeHealth(randomNodeHealth(), random.nextLong()));
     doReturn(map).when(hazelcastClient).getReplicatedMap(MAP_SQ_HEALTH_STATE);
     String uuid = randomAlphanumeric(5);
     when(hazelcastClient.getUUID()).thenReturn(uuid);