Browse Source

SONAR-15768 prometheus metric for compute engine status

tags/9.3.0.51899
Pierre 2 years ago
parent
commit
6e0a3cf998

+ 60
- 0
server/sonar-webserver-monitoring/src/main/java/org/sonar/server/monitoring/ComputeEngineMetricStatusTask.java View File

@@ -0,0 +1,60 @@
/*
* SonarQube
* Copyright (C) 2009-2021 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.monitoring;

import org.sonar.api.config.Configuration;
import org.sonar.server.app.ProcessCommandWrapper;

public class ComputeEngineMetricStatusTask implements MonitoringTask {

private static final String DELAY_IN_MILISECONDS_PROPERTY = "sonar.server.monitoring.other.initial.delay";
private static final String PERIOD_IN_MILISECONDS_PROPERTY = "sonar.server.monitoring.other.period";

private final ServerMonitoringMetrics serverMonitoringMetrics;
private final ProcessCommandWrapper processCommandWrapper;
private final Configuration config;

public ComputeEngineMetricStatusTask(ServerMonitoringMetrics serverMonitoringMetrics,
ProcessCommandWrapper processCommandWrapper, Configuration configuration) {
this.serverMonitoringMetrics = serverMonitoringMetrics;
this.processCommandWrapper = processCommandWrapper;
this.config = configuration;
}

@Override
public void run() {
if (processCommandWrapper.isCeOperational()) {
serverMonitoringMetrics.setComputeEngineStatusToGreen();
} else {
serverMonitoringMetrics.setComputeEngineStatusToRed();
}
}

@Override
public long getDelay() {
return config.getLong(DELAY_IN_MILISECONDS_PROPERTY).orElse(10_000L);
}

@Override
public long getPeriod() {
return config.getLong(PERIOD_IN_MILISECONDS_PROPERTY).orElse(10_000L);
}

}

+ 78
- 0
server/sonar-webserver-monitoring/src/main/java/org/sonar/server/monitoring/ElasticSearchMetricStatusTask.java View File

@@ -0,0 +1,78 @@
/*
* SonarQube
* Copyright (C) 2009-2021 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.monitoring;

import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.sonar.api.config.Configuration;
import org.sonar.api.utils.log.Logger;
import org.sonar.api.utils.log.Loggers;
import org.sonar.server.es.EsClient;

public class ElasticSearchMetricStatusTask implements MonitoringTask {

private static final Logger LOG = Loggers.get(ElasticSearchMetricStatusTask.class);

private static final String DELAY_IN_MILISECONDS_PROPERTY = "sonar.server.monitoring.other.initial.delay";
private static final String PERIOD_IN_MILISECONDS_PROPERTY = "sonar.server.monitoring.other.period";

private final ServerMonitoringMetrics serverMonitoringMetrics;
private final EsClient esClient;
private final Configuration config;

public ElasticSearchMetricStatusTask(ServerMonitoringMetrics serverMonitoringMetrics, EsClient esClient, Configuration configuration) {
this.serverMonitoringMetrics = serverMonitoringMetrics;
this.esClient = esClient;
config = configuration;
}

@Override
public void run() {
try {
ClusterHealthStatus esStatus = esClient.clusterHealth(new ClusterHealthRequest()).getStatus();
if (esStatus == null) {
serverMonitoringMetrics.setElasticSearchStatusToRed();
return;
}
switch (esStatus) {
case GREEN:
case YELLOW:
serverMonitoringMetrics.setElasticSearchStatusToGreen();
break;
case RED:
serverMonitoringMetrics.setElasticSearchStatusToRed();
break;
}
} catch (Exception e) {
LOG.error("Failed to query ES status", e);
serverMonitoringMetrics.setElasticSearchStatusToRed();
}
}

@Override
public long getDelay() {
return config.getLong(DELAY_IN_MILISECONDS_PROPERTY).orElse(10_000L);
}

@Override
public long getPeriod() {
return config.getLong(PERIOD_IN_MILISECONDS_PROPERTY).orElse(10_000L);
}
}

+ 28
- 0
server/sonar-webserver-monitoring/src/main/java/org/sonar/server/monitoring/ServerMonitoringMetrics.java View File

@@ -30,6 +30,8 @@ public class ServerMonitoringMetrics {
private final Gauge gitlabConfigOk;
private final Gauge bitbucketConfigOk;
private final Gauge azureConfigOk;
private final Gauge computeEngineGauge;
private final Gauge elasticsearchGauge;

private final Gauge cePendingTasksTotal;
private final Summary ceTasksRunningDuration;
@@ -65,6 +67,17 @@ public class ServerMonitoringMetrics {
.help("Compute engine task running time in seconds")
.labelNames("task_type", "project_key")
.register();

computeEngineGauge = Gauge.build()
.name("sonarqube_heath_compute_engine_status")
.help("Tells whether Compute Engine is up (healthy, ready to take tasks) or down. 0 for up, 1 for down")
.register();

elasticsearchGauge = Gauge.build()
.name("sonarqube_heath_elasticsearch_status")
.help("Tells whether Elasticsearch is up or down. 0 for Up, 1 for down")
.register();

}

public void setGithubStatusToGreen() {
@@ -107,4 +120,19 @@ public class ServerMonitoringMetrics {
ceTasksRunningDuration.labels(taskType, projectKey).observe(durationInSeconds);
}

public void setComputeEngineStatusToGreen() {
computeEngineGauge.set(0);
}

public void setComputeEngineStatusToRed() {
computeEngineGauge.set(1);
}

public void setElasticSearchStatusToGreen() {
elasticsearchGauge.set(0);
}

public void setElasticSearchStatusToRed() {
elasticsearchGauge.set(1);
}
}

+ 75
- 0
server/sonar-webserver-monitoring/src/test/java/org/sonar/server/monitoring/ComputeEngineMetricStatusTaskTest.java View File

@@ -0,0 +1,75 @@
/*
* SonarQube
* Copyright (C) 2009-2021 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.monitoring;

import io.prometheus.client.CollectorRegistry;
import org.assertj.core.api.Assertions;
import org.junit.Before;
import org.junit.Test;
import org.sonar.api.config.Configuration;
import org.sonar.api.config.internal.MapSettings;
import org.sonar.server.app.ProcessCommandWrapper;

import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;

public class ComputeEngineMetricStatusTaskTest {

private final ServerMonitoringMetrics serverMonitoringMetrics = mock(ServerMonitoringMetrics.class);
private final ProcessCommandWrapper processCommandWrapper = mock(ProcessCommandWrapper.class);
private final Configuration configuration = new MapSettings().asConfig();

private final ComputeEngineMetricStatusTask underTest = new ComputeEngineMetricStatusTask(serverMonitoringMetrics, processCommandWrapper, configuration);

@Before
public void before() {
CollectorRegistry.defaultRegistry.clear();
}

@Test
public void when_compute_engine_up_status_is_updated_to_green() {
when(processCommandWrapper.isCeOperational()).thenReturn(true);

underTest.run();

verify(serverMonitoringMetrics, times(1)).setComputeEngineStatusToGreen();
verifyNoMoreInteractions(serverMonitoringMetrics);
}

@Test
public void when_compute_engine_down_status_is_updated_to_red() {
when(processCommandWrapper.isCeOperational()).thenReturn(false);

underTest.run();

verify(serverMonitoringMetrics, times(1)).setComputeEngineStatusToRed();
verifyNoMoreInteractions(serverMonitoringMetrics);
}

@Test
public void task_has_default_delay(){
Assertions.assertThat(underTest.getDelay()).isPositive();
Assertions.assertThat(underTest.getPeriod()).isPositive();
}

}

+ 127
- 0
server/sonar-webserver-monitoring/src/test/java/org/sonar/server/monitoring/ElasticSearchMetricStatusTaskTest.java View File

@@ -0,0 +1,127 @@
/*
* SonarQube
* Copyright (C) 2009-2021 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.monitoring;

import io.prometheus.client.CollectorRegistry;
import org.assertj.core.api.Assertions;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.mockito.Mockito;
import org.sonar.api.config.Configuration;
import org.sonar.api.config.internal.MapSettings;
import org.sonar.api.utils.log.LogTester;
import org.sonar.api.utils.log.LoggerLevel;
import org.sonar.server.es.EsClient;

import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;

public class ElasticSearchMetricStatusTaskTest {

@Rule
public LogTester logTester = new LogTester();

private final ServerMonitoringMetrics serverMonitoringMetrics = mock(ServerMonitoringMetrics.class);
private final EsClient esClient = mock(EsClient.class);
private final Configuration configuration = new MapSettings().asConfig();

private final ElasticSearchMetricStatusTask underTest = new ElasticSearchMetricStatusTask(serverMonitoringMetrics, esClient, configuration);

@Before
public void before() {
CollectorRegistry.defaultRegistry.clear();
}

@Test
public void when_elasticsearch_up_status_is_updated_to_green() {
ClusterHealthResponse clusterHealthResponse = new ClusterHealthResponse();
clusterHealthResponse.setStatus(ClusterHealthStatus.GREEN);
when(esClient.clusterHealth(any())).thenReturn(clusterHealthResponse);

underTest.run();

verify(serverMonitoringMetrics, times(1)).setElasticSearchStatusToGreen();
verifyNoMoreInteractions(serverMonitoringMetrics);
}

@Test
public void when_elasticsearch_yellow_status_is_updated_to_green() {
ClusterHealthResponse clusterHealthResponse = new ClusterHealthResponse();
clusterHealthResponse.setStatus(ClusterHealthStatus.YELLOW);
when(esClient.clusterHealth(any())).thenReturn(clusterHealthResponse);

underTest.run();

verify(serverMonitoringMetrics, times(1)).setElasticSearchStatusToGreen();
verifyNoMoreInteractions(serverMonitoringMetrics);
}

@Test
public void when_elasticsearch_down_status_is_updated_to_red() {
ClusterHealthResponse clusterHealthResponse = new ClusterHealthResponse();
clusterHealthResponse.setStatus(ClusterHealthStatus.RED);
when(esClient.clusterHealth(any())).thenReturn(clusterHealthResponse);

underTest.run();

verify(serverMonitoringMetrics, times(1)).setElasticSearchStatusToRed();
verifyNoMoreInteractions(serverMonitoringMetrics);
}

@Test
public void when_no_es_status_null_status_is_updated_to_red() {
ClusterHealthResponse clusterHealthResponse = Mockito.mock(ClusterHealthResponse.class);
when(clusterHealthResponse.getStatus()).thenReturn(null);
when(esClient.clusterHealth(any())).thenReturn(clusterHealthResponse);

underTest.run();

verify(serverMonitoringMetrics, times(1)).setElasticSearchStatusToRed();
verifyNoMoreInteractions(serverMonitoringMetrics);
}

@Test
public void when_es_status_throw_exception_status_is_updated_to_red() {
when(esClient.clusterHealth(any())).thenThrow(new IllegalStateException("exception in cluster health"));

underTest.run();

verify(serverMonitoringMetrics, times(1)).setElasticSearchStatusToRed();
verifyNoMoreInteractions(serverMonitoringMetrics);

assertThat(logTester.logs()).hasSize(1);
assertThat(logTester.logs(LoggerLevel.ERROR)).containsOnly("Failed to query ES status");
}

@Test
public void task_has_default_delay(){
Assertions.assertThat(underTest.getDelay()).isPositive();
Assertions.assertThat(underTest.getPeriod()).isPositive();
}

}

+ 8
- 1
server/sonar-webserver-monitoring/src/test/java/org/sonar/server/monitoring/ServerMonitoringMetricsTest.java View File

@@ -23,7 +23,6 @@ import io.prometheus.client.Collector;
import io.prometheus.client.CollectorRegistry;
import java.util.Collections;
import java.util.Enumeration;
import java.util.Iterator;
import org.junit.Before;
import org.junit.Test;

@@ -54,11 +53,15 @@ public class ServerMonitoringMetricsTest {
metrics.setGitlabStatusToGreen();
metrics.setAzureStatusToGreen();
metrics.setBitbucketStatusToGreen();
metrics.setComputeEngineStatusToGreen();
metrics.setElasticSearchStatusToGreen();

assertThat(CollectorRegistry.defaultRegistry.getSampleValue("github_config_ok")).isZero();
assertThat(CollectorRegistry.defaultRegistry.getSampleValue("gitlab_config_ok")).isZero();
assertThat(CollectorRegistry.defaultRegistry.getSampleValue("bitbucket_config_ok")).isZero();
assertThat(CollectorRegistry.defaultRegistry.getSampleValue("azure_config_ok")).isZero();
assertThat(CollectorRegistry.defaultRegistry.getSampleValue("sonarqube_heath_compute_engine_status")).isZero();
assertThat(CollectorRegistry.defaultRegistry.getSampleValue("sonarqube_heath_elasticsearch_status")).isZero();
}

@Test
@@ -69,11 +72,15 @@ public class ServerMonitoringMetricsTest {
metrics.setGitlabStatusToRed();
metrics.setAzureStatusToRed();
metrics.setBitbucketStatusToRed();
metrics.setComputeEngineStatusToRed();
metrics.setElasticSearchStatusToRed();

assertThat(CollectorRegistry.defaultRegistry.getSampleValue("github_config_ok")).isEqualTo(1);
assertThat(CollectorRegistry.defaultRegistry.getSampleValue("gitlab_config_ok")).isEqualTo(1);
assertThat(CollectorRegistry.defaultRegistry.getSampleValue("bitbucket_config_ok")).isEqualTo(1);
assertThat(CollectorRegistry.defaultRegistry.getSampleValue("azure_config_ok")).isEqualTo(1);
assertThat(CollectorRegistry.defaultRegistry.getSampleValue("sonarqube_heath_compute_engine_status")).isEqualTo(1);
assertThat(CollectorRegistry.defaultRegistry.getSampleValue("sonarqube_heath_elasticsearch_status")).isEqualTo(1);
}

@Test

+ 1
- 1
server/sonar-webserver-webapi/src/main/java/org/sonar/server/platform/ws/SafeModeMonitoringMetricAction.java View File

@@ -37,7 +37,7 @@ import static java.nio.charset.StandardCharsets.UTF_8;

public class SafeModeMonitoringMetricAction implements MonitoringWsAction {

protected static final Gauge isWebUpGauge = Gauge.build().name("is_web_up").help("Tells whether web service is up").register();
protected static final Gauge isWebUpGauge = Gauge.build().name("sonarqube_heath_web_status").help("Tells whether web service is up").register();

private final SystemPasscode systemPasscode;
private final BearerPasscode bearerPasscode;

+ 9
- 3
server/sonar-webserver-webapi/src/main/resources/org/sonar/server/monitoring/monitoring-metrics.txt View File

@@ -1,3 +1,9 @@
# HELP is_web_up Tells whether web service is up
# TYPE is_web_up gauge
is_web_up 0.0
# HELP sonarqube_heath_web_status Tells whether web service is up
# TYPE sonarqube_heath_web_status gauge
sonarqube_heath_web_status 0.0
# HELP sonarqube_heath_compute_engine_status Tells whether Compute Engine is up (healthy, ready to take tasks) or down. 0 for up, 1 for down
# TYPE sonarqube_heath_compute_engine_status gauge
sonarqube_heath_compute_engine_status 0.0
# HELP sonarqube_heath_elasticsearch_status Tells whether Elasticsearch is up or down. 0 for Up, 1 for down
# TYPE sonarqube_heath_elasticsearch_status gauge
sonarqube_heath_elasticsearch_status 0.0

+ 9
- 9
server/sonar-webserver-webapi/src/test/java/org/sonar/server/monitoring/MetricsActionTest.java View File

@@ -86,9 +86,9 @@ public class MetricsActionTest {
TestResponse response = ws.newRequest().execute();
String content = response.getInput();
assertThat(content)
.contains("# HELP is_web_up Tells whether web service is up")
.contains("# TYPE is_web_up gauge")
.contains("is_web_up 0.0");
.contains("# HELP sonarqube_heath_web_status Tells whether web service is up")
.contains("# TYPE sonarqube_heath_web_status gauge")
.contains("sonarqube_heath_web_status 0.0");
}

@Test
@@ -98,9 +98,9 @@ public class MetricsActionTest {
TestResponse response = ws.newRequest().execute();
String content = response.getInput();
assertThat(content)
.contains("# HELP is_web_up Tells whether web service is up")
.contains("# TYPE is_web_up gauge")
.contains("is_web_up 0.0");
.contains("# HELP sonarqube_heath_web_status Tells whether web service is up")
.contains("# TYPE sonarqube_heath_web_status gauge")
.contains("sonarqube_heath_web_status 0.0");
}

@Test
@@ -110,9 +110,9 @@ public class MetricsActionTest {
TestResponse response = ws.newRequest().execute();
String content = response.getInput();
assertThat(content)
.contains("# HELP is_web_up Tells whether web service is up")
.contains("# TYPE is_web_up gauge")
.contains("is_web_up 0.0");
.contains("# HELP sonarqube_heath_web_status Tells whether web service is up")
.contains("# TYPE sonarqube_heath_web_status gauge")
.contains("sonarqube_heath_web_status 0.0");
}

}

+ 6
- 6
server/sonar-webserver-webapi/src/test/java/org/sonar/server/platform/ws/SafeModeMonitoringMetricActionTest.java View File

@@ -76,9 +76,9 @@ public class SafeModeMonitoringMetricActionTest {
TestResponse response = ws.newRequest().execute();
String content = response.getInput();
assertThat(content)
.contains("# HELP is_web_up Tells whether web service is up")
.contains("# TYPE is_web_up gauge")
.contains("is_web_up 0.0");
.contains("# HELP sonarqube_heath_web_status Tells whether web service is up")
.contains("# TYPE sonarqube_heath_web_status gauge")
.contains("sonarqube_heath_web_status 0.0");
}

@Test
@@ -88,9 +88,9 @@ public class SafeModeMonitoringMetricActionTest {
TestResponse response = ws.newRequest().execute();
String content = response.getInput();
assertThat(content)
.contains("# HELP is_web_up Tells whether web service is up")
.contains("# TYPE is_web_up gauge")
.contains("is_web_up 0.0");
.contains("# HELP sonarqube_heath_web_status Tells whether web service is up")
.contains("# TYPE sonarqube_heath_web_status gauge")
.contains("sonarqube_heath_web_status 0.0");
}

@Test

+ 6
- 1
server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java View File

@@ -126,13 +126,15 @@ import org.sonar.server.measure.ws.MeasuresWsModule;
import org.sonar.server.metric.MetricFinder;
import org.sonar.server.metric.UnanalyzedLanguageMetrics;
import org.sonar.server.metric.ws.MetricsWsModule;
import org.sonar.server.monitoring.ComputeEngineMetricStatusTask;
import org.sonar.server.monitoring.ElasticSearchMetricStatusTask;
import org.sonar.server.monitoring.MainCollector;
import org.sonar.server.monitoring.MonitoringWsModule;
import org.sonar.server.monitoring.ServerMonitoringMetrics;
import org.sonar.server.monitoring.ce.NumberOfTasksInQueueTask;
import org.sonar.server.monitoring.ce.RecentTasksDurationTask;
import org.sonar.server.monitoring.devops.AzureMetricsTask;
import org.sonar.server.monitoring.devops.BitbucketMetricsTask;
import org.sonar.server.monitoring.ServerMonitoringMetrics;
import org.sonar.server.monitoring.devops.GithubMetricsTask;
import org.sonar.server.monitoring.devops.GitlabMetricsTask;
import org.sonar.server.newcodeperiod.ws.NewCodePeriodsWsModule;
@@ -593,6 +595,9 @@ public class PlatformLevel4 extends PlatformLevel {
NumberOfTasksInQueueTask.class,
RecentTasksDurationTask.class,

ComputeEngineMetricStatusTask.class,
ElasticSearchMetricStatusTask.class,

MainCollector.class,

PluginsRiskConsentFilter.class

Loading…
Cancel
Save