diff options
7 files changed, 166 insertions, 20 deletions
diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/response/NodeStats.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/NodeStats.java index 865754a5455..c7650973444 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/response/NodeStats.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/response/NodeStats.java @@ -34,6 +34,7 @@ public class NodeStats { private final long openFileDescriptors; private final long maxFileDescriptors; private final long diskAvailableBytes; + private final long diskTotalBytes; private final long fieldDataCircuitBreakerLimit; private final long fieldDataCircuitBreakerEstimation; @@ -51,6 +52,7 @@ public class NodeStats { this.openFileDescriptors = nodeStatsJson.getAsJsonObject(PROCESS_ATTRIBUTE_NAME).get("open_file_descriptors").getAsLong(); this.maxFileDescriptors = nodeStatsJson.getAsJsonObject(PROCESS_ATTRIBUTE_NAME).get("max_file_descriptors").getAsLong(); this.diskAvailableBytes = nodeStatsJson.getAsJsonObject("fs").getAsJsonObject("total").get("available_in_bytes").getAsLong(); + this.diskTotalBytes = nodeStatsJson.getAsJsonObject("fs").getAsJsonObject("total").get("total_in_bytes").getAsLong(); this.fieldDataCircuitBreakerLimit = nodeStatsJson.getAsJsonObject(BREAKERS_ATTRIBUTE_NAME).getAsJsonObject("fielddata").get("limit_size_in_bytes").getAsLong(); this.fieldDataCircuitBreakerEstimation = nodeStatsJson.getAsJsonObject(BREAKERS_ATTRIBUTE_NAME).getAsJsonObject("fielddata").get("estimated_size_in_bytes").getAsLong(); @@ -89,6 +91,10 @@ public class NodeStats { return diskAvailableBytes; } + public long getDiskTotalBytes() { + return diskTotalBytes; + } + public long getFieldDataCircuitBreakerLimit() { return fieldDataCircuitBreakerLimit; } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/response/NodeStatsResponseTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/response/NodeStatsResponseTest.java index 0b234fea25a..1d8c70384c9 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/response/NodeStatsResponseTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/response/NodeStatsResponseTest.java @@ -105,6 +105,7 @@ public class NodeStatsResponseTest { assertThat(nodeStats.getOpenFileDescriptors()).isEqualTo(296); assertThat(nodeStats.getMaxFileDescriptors()).isEqualTo(10240); assertThat(nodeStats.getDiskAvailableBytes()).isEqualTo(136144027648L); + assertThat(nodeStats.getDiskTotalBytes()).isEqualTo(250685575168L); assertThat(nodeStats.getFieldDataCircuitBreakerLimit()).isEqualTo(207591833); assertThat(nodeStats.getFieldDataCircuitBreakerEstimation()).isEqualTo(4880); diff --git a/server/sonar-webserver-monitoring/src/main/java/org/sonar/server/monitoring/ElasticSearchMetricStatusTask.java b/server/sonar-webserver-monitoring/src/main/java/org/sonar/server/monitoring/ElasticSearchMetricTask.java index 58fa6b59bf0..5c9b7de0302 100644 --- a/server/sonar-webserver-monitoring/src/main/java/org/sonar/server/monitoring/ElasticSearchMetricStatusTask.java +++ b/server/sonar-webserver-monitoring/src/main/java/org/sonar/server/monitoring/ElasticSearchMetricTask.java @@ -25,19 +25,21 @@ import org.sonar.api.config.Configuration; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; import org.sonar.server.es.EsClient; +import org.sonar.server.es.response.NodeStats; +import org.sonar.server.es.response.NodeStatsResponse; -public class ElasticSearchMetricStatusTask implements MonitoringTask { +public class ElasticSearchMetricTask implements MonitoringTask { - private static final Logger LOG = Loggers.get(ElasticSearchMetricStatusTask.class); + private static final Logger LOG = Loggers.get(ElasticSearchMetricTask.class); - private static final String DELAY_IN_MILISECONDS_PROPERTY = "sonar.server.monitoring.other.initial.delay"; - private static final String PERIOD_IN_MILISECONDS_PROPERTY = "sonar.server.monitoring.other.period"; + private static final String DELAY_IN_MILISECONDS_PROPERTY = "sonar.server.monitoring.es.initial.delay"; + private static final String PERIOD_IN_MILISECONDS_PROPERTY = "sonar.server.monitoring.es.period"; private final ServerMonitoringMetrics serverMonitoringMetrics; private final EsClient esClient; private final Configuration config; - public ElasticSearchMetricStatusTask(ServerMonitoringMetrics serverMonitoringMetrics, EsClient esClient, Configuration configuration) { + public ElasticSearchMetricTask(ServerMonitoringMetrics serverMonitoringMetrics, EsClient esClient, Configuration configuration) { this.serverMonitoringMetrics = serverMonitoringMetrics; this.esClient = esClient; config = configuration; @@ -45,20 +47,25 @@ public class ElasticSearchMetricStatusTask implements MonitoringTask { @Override public void run() { + updateElasticSearchHealthStatus(); + updateFileSystemMetrics(); + } + + private void updateElasticSearchHealthStatus() { try { ClusterHealthStatus esStatus = esClient.clusterHealth(new ClusterHealthRequest()).getStatus(); if (esStatus == null) { serverMonitoringMetrics.setElasticSearchStatusToRed(); - return; - } - switch (esStatus) { - case GREEN: - case YELLOW: - serverMonitoringMetrics.setElasticSearchStatusToGreen(); - break; - case RED: - serverMonitoringMetrics.setElasticSearchStatusToRed(); - break; + } else { + switch (esStatus) { + case GREEN: + case YELLOW: + serverMonitoringMetrics.setElasticSearchStatusToGreen(); + break; + case RED: + serverMonitoringMetrics.setElasticSearchStatusToRed(); + break; + } } } catch (Exception e) { LOG.error("Failed to query ES status", e); @@ -66,6 +73,22 @@ public class ElasticSearchMetricStatusTask implements MonitoringTask { } } + private void updateFileSystemMetrics() { + try { + NodeStatsResponse nodeStatsResponse = esClient.nodesStats(); + if (nodeStatsResponse.getNodeStats().isEmpty()) { + LOG.error("Failed to query ES status, no nodes stats returned by elasticsearch API"); + } else { + for (NodeStats nodeStat : nodeStatsResponse.getNodeStats()) { + serverMonitoringMetrics.setElasticSearchDiskSpaceFreeBytes(nodeStat.getName(), nodeStat.getDiskAvailableBytes()); + serverMonitoringMetrics.setElasticSearchDiskSpaceTotalBytes(nodeStat.getName(), nodeStat.getDiskTotalBytes()); + } + } + } catch (Exception e) { + LOG.error("Failed to query ES status", e); + } + } + @Override public long getDelay() { return config.getLong(DELAY_IN_MILISECONDS_PROPERTY).orElse(10_000L); diff --git a/server/sonar-webserver-monitoring/src/main/java/org/sonar/server/monitoring/ServerMonitoringMetrics.java b/server/sonar-webserver-monitoring/src/main/java/org/sonar/server/monitoring/ServerMonitoringMetrics.java index d757d4fd66c..0a7ee64a457 100644 --- a/server/sonar-webserver-monitoring/src/main/java/org/sonar/server/monitoring/ServerMonitoringMetrics.java +++ b/server/sonar-webserver-monitoring/src/main/java/org/sonar/server/monitoring/ServerMonitoringMetrics.java @@ -37,6 +37,8 @@ public class ServerMonitoringMetrics { private final Gauge cePendingTasksTotal; private final Summary ceTasksRunningDuration; + private final Gauge elasticsearchDiskSpaceFreeBytesGauge; + private final Gauge elasticSearchDiskSpaceTotalBytes; private final Gauge licenseDaysBeforeExpiration; private final Gauge linesOfCodeRemaining; @@ -98,6 +100,18 @@ public class ServerMonitoringMetrics { .name("sonarqube_license_number_of_lines_analyzed_total") .help("Number of lines analyzed.") .register(); + + elasticsearchDiskSpaceFreeBytesGauge = Gauge.build() + .name("sonarqube_elasticsearch_disk_space_free_bytes") + .help("Space left on device") + .labelNames("node_name") + .register(); + + elasticSearchDiskSpaceTotalBytes = Gauge.build() + .name("sonarqube_elasticsearch_disk_space_total_bytes") + .help("Total disk space on the device") + .labelNames("node_name") + .register(); } public void setGithubStatusToGreen() { @@ -167,4 +181,12 @@ public class ServerMonitoringMetrics { public void setLinesOfCodeAnalyzed(long loc) { linesOfCodeAnalyzed.set(loc); } + + public void setElasticSearchDiskSpaceFreeBytes(String name, long diskAvailableBytes) { + elasticsearchDiskSpaceFreeBytesGauge.labels(name).set(diskAvailableBytes); + } + + public void setElasticSearchDiskSpaceTotalBytes(String name, long disktotalBytes) { + elasticSearchDiskSpaceTotalBytes.labels(name).set(disktotalBytes); + } } diff --git a/server/sonar-webserver-monitoring/src/test/java/org/sonar/server/monitoring/ElasticSearchMetricStatusTaskTest.java b/server/sonar-webserver-monitoring/src/test/java/org/sonar/server/monitoring/ElasticSearchMetricTaskTest.java index e678de8d659..46c1c60ffdb 100644 --- a/server/sonar-webserver-monitoring/src/test/java/org/sonar/server/monitoring/ElasticSearchMetricStatusTaskTest.java +++ b/server/sonar-webserver-monitoring/src/test/java/org/sonar/server/monitoring/ElasticSearchMetricTaskTest.java @@ -19,7 +19,12 @@ */ package org.sonar.server.monitoring; +import com.google.gson.Gson; +import com.google.gson.JsonObject; import io.prometheus.client.CollectorRegistry; +import java.io.IOException; +import java.net.URL; +import java.nio.charset.StandardCharsets; import org.assertj.core.api.Assertions; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.cluster.health.ClusterHealthStatus; @@ -29,9 +34,12 @@ import org.junit.Test; import org.mockito.Mockito; import org.sonar.api.config.Configuration; import org.sonar.api.config.internal.MapSettings; +import org.sonar.api.internal.apachecommons.io.IOUtils; +import org.sonar.api.internal.apachecommons.lang.StringUtils; import org.sonar.api.utils.log.LogTester; import org.sonar.api.utils.log.LoggerLevel; import org.sonar.server.es.EsClient; +import org.sonar.server.es.response.NodeStatsResponse; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; @@ -41,7 +49,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; -public class ElasticSearchMetricStatusTaskTest { +public class ElasticSearchMetricTaskTest { @Rule public LogTester logTester = new LogTester(); @@ -50,7 +58,7 @@ public class ElasticSearchMetricStatusTaskTest { private final EsClient esClient = mock(EsClient.class); private final Configuration configuration = new MapSettings().asConfig(); - private final ElasticSearchMetricStatusTask underTest = new ElasticSearchMetricStatusTask(serverMonitoringMetrics, esClient, configuration); + private final ElasticSearchMetricTask underTest = new ElasticSearchMetricTask(serverMonitoringMetrics, esClient, configuration); @Before public void before() { @@ -69,6 +77,29 @@ public class ElasticSearchMetricStatusTaskTest { verifyNoMoreInteractions(serverMonitoringMetrics); } + + + @Test + public void elasticsearch_free_disk_space_is_updated() throws IOException { + URL esNodeResponseUrl = getClass().getResource("es-node-response.json"); + String jsonPayload = StringUtils.trim(IOUtils.toString(esNodeResponseUrl, StandardCharsets.UTF_8)); + + JsonObject jsonObject = new Gson().fromJson(jsonPayload, JsonObject.class); + NodeStatsResponse nodeStats = NodeStatsResponse.toNodeStatsResponse(jsonObject); + + when(esClient.nodesStats()).thenReturn(nodeStats); + + underTest.run(); + + String nodeName = nodeStats.getNodeStats().get(0).getName(); + verify(serverMonitoringMetrics, times(1)).setElasticSearchDiskSpaceFreeBytes(nodeName, 136144027648L); + verify(serverMonitoringMetrics, times(1)).setElasticSearchDiskSpaceTotalBytes(nodeName, 250685575168L); + + // elasticsearch health status is not mocked in this test, so this part raise an exception + assertThat(logTester.logs()).hasSize(1); + assertThat(logTester.logs(LoggerLevel.ERROR)).containsOnly("Failed to query ES status"); + } + @Test public void when_elasticsearch_yellow_status_is_updated_to_green() { ClusterHealthResponse clusterHealthResponse = new ClusterHealthResponse(); @@ -114,7 +145,7 @@ public class ElasticSearchMetricStatusTaskTest { verify(serverMonitoringMetrics, times(1)).setElasticSearchStatusToRed(); verifyNoMoreInteractions(serverMonitoringMetrics); - assertThat(logTester.logs()).hasSize(1); + assertThat(logTester.logs()).hasSize(2); assertThat(logTester.logs(LoggerLevel.ERROR)).containsOnly("Failed to query ES status"); } diff --git a/server/sonar-webserver-monitoring/src/test/resources/org/sonar/server/monitoring/es-node-response.json b/server/sonar-webserver-monitoring/src/test/resources/org/sonar/server/monitoring/es-node-response.json new file mode 100644 index 00000000000..acb686b6a3d --- /dev/null +++ b/server/sonar-webserver-monitoring/src/test/resources/org/sonar/server/monitoring/es-node-response.json @@ -0,0 +1,63 @@ +{ + "nodes": { + "YnKPZcbGRamRQGxjErLWoQ": { + "name": "sonarqube", + "host": "127.0.0.1", + "indices": { + "docs": { + "count": 13557 + }, + "store": { + "size_in_bytes": 8670970 + }, + "query_cache": { + "memory_size_in_bytes": 0 + }, + "fielddata": { + "memory_size_in_bytes": 4880 + }, + "translog": { + "size_in_bytes": 8274137 + }, + "request_cache": { + "memory_size_in_bytes": 0 + } + }, + "process": { + "open_file_descriptors": 296, + "max_file_descriptors": 10240, + "cpu": { + "percent": 7 + } + }, + "jvm": { + "mem": { + "heap_used_in_bytes": 158487160, + "heap_used_percent": 30, + "heap_max_in_bytes": 518979584, + "non_heap_used_in_bytes": 109066592 + }, + "threads": { + "count": 70 + } + }, + "fs": { + "total": { + "total_in_bytes": 250685575168, + "free_in_bytes": 142843138048, + "available_in_bytes": 136144027648 + } + }, + "breakers": { + "request": { + "limit_size_in_bytes": 311387750, + "estimated_size_in_bytes": 1 + }, + "fielddata": { + "limit_size_in_bytes": 207591833, + "estimated_size_in_bytes": 4880 + } + } + } + } +}
\ No newline at end of file diff --git a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java index 3d7b0f2b5fa..17566876094 100644 --- a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java +++ b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java @@ -127,7 +127,7 @@ import org.sonar.server.metric.MetricFinder; import org.sonar.server.metric.UnanalyzedLanguageMetrics; import org.sonar.server.metric.ws.MetricsWsModule; import org.sonar.server.monitoring.ComputeEngineMetricStatusTask; -import org.sonar.server.monitoring.ElasticSearchMetricStatusTask; +import org.sonar.server.monitoring.ElasticSearchMetricTask; import org.sonar.server.monitoring.MainCollector; import org.sonar.server.monitoring.MonitoringWsModule; import org.sonar.server.monitoring.ServerMonitoringMetrics; @@ -596,7 +596,7 @@ public class PlatformLevel4 extends PlatformLevel { RecentTasksDurationTask.class, ComputeEngineMetricStatusTask.class, - ElasticSearchMetricStatusTask.class, + ElasticSearchMetricTask.class, MainCollector.class, |