Browse Source

SONAR-20787 Add telemetry about quality gate conditions and sonar way (#9743)

tags/10.3.0.82913
Nolwenn Cadic 7 months ago
parent
commit
2eaf4da9b1

+ 15
- 0
server/sonar-db-dao/src/it/java/org/sonar/db/qualitygate/QualityGateConditionDaoIT.java View File

@@ -33,6 +33,7 @@ import org.sonar.db.metric.MetricDto;

import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.AssertionsForClassTypes.tuple;

public class QualityGateConditionDaoIT {

@@ -81,6 +82,20 @@ public class QualityGateConditionDaoIT {
assertThat(underTest.selectForQualityGate(dbSession, "5")).isEmpty();
}

@Test
public void selectAll() {
MetricDto metric = dbTester.measures().insertMetric(t -> t.setEnabled(true));
QualityGateConditionDto condition1 = insertQGCondition("uuid1", metric.getUuid());
QualityGateConditionDto condition2 = insertQGCondition("uuid2", metric.getUuid());
QualityGateConditionDto condition3 = insertQGCondition("uuid3", metric.getUuid());

assertThat(underTest.selectAll(dbSession))
.extracting(QualityGateConditionDto::getUuid, QualityGateConditionDto::getMetricUuid)
.containsOnly(tuple(condition1.getUuid(), condition1.getMetricUuid()),
tuple(condition2.getUuid(), condition2.getMetricUuid()),
tuple(condition3.getUuid(), condition3.getMetricUuid()));
}

@Test
public void testSelectByUuid() {
QualityGateConditionDto condition = insertQGCondition("1", "2", "GT", "20");

+ 4
- 0
server/sonar-db-dao/src/main/java/org/sonar/db/qualitygate/QualityGateConditionDao.java View File

@@ -34,6 +34,10 @@ public class QualityGateConditionDao implements Dao {
return mapper(session).selectForQualityGate(qGateUuid);
}

public Collection<QualityGateConditionDto> selectAll(DbSession session) {
return mapper(session).selectAll();
}

public QualityGateConditionDto selectByUuid(String uuid, DbSession session) {
return mapper(session).selectByUuid(uuid);
}

+ 2
- 0
server/sonar-db-dao/src/main/java/org/sonar/db/qualitygate/QualityGateConditionMapper.java View File

@@ -27,6 +27,8 @@ public interface QualityGateConditionMapper {

List<QualityGateConditionDto> selectForQualityGate(String qGateUuid);

List<QualityGateConditionDto> selectAll();

void update(QualityGateConditionDto newCondition);

QualityGateConditionDto selectByUuid(String uuid);

+ 7
- 0
server/sonar-db-dao/src/main/resources/org/sonar/db/qualitygate/QualityGateConditionMapper.xml View File

@@ -20,6 +20,13 @@
order by created_at asc
</select>

<select id="selectAll" resultType="QualityGateCondition">
select
<include refid="conditionColumns"/>
from quality_gate_conditions
order by qgate_uuid asc
</select>

<select id="selectByUuid" parameterType="String" resultType="QualityGateCondition">
select
<include refid="conditionColumns"/>

+ 2
- 0
server/sonar-server-common/src/main/java/org/sonar/server/qualitygate/QualityGate.java View File

@@ -28,6 +28,8 @@ import static java.util.Objects.requireNonNull;

@Immutable
public class QualityGate {

public static final String BUILTIN_QUALITY_GATE_NAME = "Sonar way";
private final String id;
private final String name;
private final Set<Condition> conditions;

+ 8
- 0
server/sonar-server-common/src/main/java/org/sonar/server/qualitygate/QualityGateFinder.java View File

@@ -25,6 +25,9 @@ import org.sonar.db.DbSession;
import org.sonar.db.project.ProjectDto;
import org.sonar.db.qualitygate.QualityGateDto;

import static java.lang.String.format;
import static org.sonar.server.qualitygate.QualityGate.BUILTIN_QUALITY_GATE_NAME;

public class QualityGateFinder {
private final DbClient dbClient;

@@ -55,6 +58,11 @@ public class QualityGateFinder {
return Optional.ofNullable(dbClient.qualityGateDao().selectDefault(dbSession)).orElseThrow(() -> new IllegalStateException("Default quality gate is missing"));
}

public QualityGateDto getSonarWay(DbSession dbSession) {
return Optional.ofNullable(dbClient.qualityGateDao().selectByName(dbSession, BUILTIN_QUALITY_GATE_NAME)).orElseThrow(() ->
new IllegalStateException(format("%s quality gate is missing", BUILTIN_QUALITY_GATE_NAME)));
}

public static class QualityGateData {
private final String uuid;
private final String name;

+ 15
- 1
server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryData.java View File

@@ -31,6 +31,7 @@ import org.sonar.core.platform.EditionProvider;
import org.sonar.core.platform.EditionProvider.Edition;
import org.sonar.db.project.CreationMethod;
import org.sonar.db.user.UserTelemetryDto;
import org.sonar.server.qualitygate.Condition;

import static java.util.Objects.requireNonNullElse;
import static org.sonar.db.newcodeperiod.NewCodePeriodType.PREVIOUS_VERSION;
@@ -43,6 +44,7 @@ public class TelemetryData {
private final Database database;
private final EditionProvider.Edition edition;
private final String defaultQualityGate;
private final String sonarWayQualityGate;
private final Long installationDate;
private final String installationVersion;
private final boolean inContainer;
@@ -68,6 +70,7 @@ public class TelemetryData {
database = builder.database;
edition = builder.edition;
defaultQualityGate = builder.defaultQualityGate;
sonarWayQualityGate = builder.sonarWayQualityGate;
installationDate = builder.installationDate;
installationVersion = builder.installationVersion;
inContainer = builder.inContainer;
@@ -114,6 +117,10 @@ public class TelemetryData {
return defaultQualityGate;
}

public String getSonarWayQualityGate() {
return sonarWayQualityGate;
}

public Long getInstallationDate() {
return installationDate;
}
@@ -190,6 +197,8 @@ public class TelemetryData {
private Database database;
private Edition edition;
private String defaultQualityGate;

private String sonarWayQualityGate;
private Long installationDate;
private String installationVersion;
private boolean inContainer = false;
@@ -246,6 +255,11 @@ public class TelemetryData {
return this;
}

Builder setSonarWayQualityGate(String sonarWayQualityGate) {
this.sonarWayQualityGate = sonarWayQualityGate;
return this;
}

Builder setInstallationDate(@Nullable Long installationDate) {
this.installationDate = installationDate;
return this;
@@ -360,7 +374,7 @@ public class TelemetryData {
record Project(String projectUuid, Long lastAnalysis, String language, String qualityProfile, Long loc) {
}

record QualityGate(String uuid, String caycStatus) {
record QualityGate(String uuid, String caycStatus, List<Condition> conditions) {
}

public record QualityProfile(String uuid, @Nullable String parentUuid, String language, boolean isDefault,

+ 11
- 0
server/sonar-server-common/src/main/java/org/sonar/server/telemetry/TelemetryDataJsonWriter.java View File

@@ -63,6 +63,7 @@ public class TelemetryDataJsonWriter {
json.prop(NCD_ID, telemetryData.getNcdId());
telemetryData.getEdition().ifPresent(e -> json.prop("edition", e.name().toLowerCase(Locale.ENGLISH)));
json.prop("defaultQualityGate", telemetryData.getDefaultQualityGate());
json.prop("sonarway_quality_gate_uuid", telemetryData.getSonarWayQualityGate());
json.name("database");
json.beginObject();
json.prop("name", telemetryData.getDatabase().name());
@@ -222,6 +223,16 @@ public class TelemetryDataJsonWriter {
json.beginObject();
json.prop("uuid", qualityGate.uuid());
json.prop("caycStatus", qualityGate.caycStatus());
json.name("conditions");
json.beginArray();
qualityGate.conditions().forEach(condition -> {
json.beginObject();
json.prop("metric", condition.getMetricKey());
json.prop("comparison_operator", condition.getOperator().getDbValue());
json.prop("error_value", condition.getErrorThreshold());
json.endObject();
});
json.endArray();
json.endObject();
});
json.endArray();

+ 1
- 1
server/sonar-server-common/src/test/java/org/sonar/server/qualitygate/ConditionTest.java View File

@@ -40,7 +40,7 @@ public class ConditionTest {
}

@Test
public void constructor_throws_NPE_if_operator_is_null() {
public void constructor_throws_NPE_if_operator_operator_is_null() {
assertThatThrownBy(() -> new Condition(METRIC_KEY, null, ERROR_THRESHOLD))
.isInstanceOf(NullPointerException.class)
.hasMessage("operator can't be null");

+ 75
- 17
server/sonar-server-common/src/test/java/org/sonar/server/telemetry/TelemetryDataJsonWriterTest.java View File

@@ -42,6 +42,7 @@ import org.sonar.core.platform.EditionProvider;
import org.sonar.core.telemetry.TelemetryExtension;
import org.sonar.db.project.CreationMethod;
import org.sonar.db.user.UserTelemetryDto;
import org.sonar.server.qualitygate.Condition;

import static java.util.stream.Collectors.joining;
import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic;
@@ -50,6 +51,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.sonar.db.newcodeperiod.NewCodePeriodType.NUMBER_OF_DAYS;
import static org.sonar.db.newcodeperiod.NewCodePeriodType.PREVIOUS_VERSION;
import static org.sonar.server.qualitygate.Condition.Operator.fromDbValue;
import static org.sonar.test.JsonAssert.assertJson;

@RunWith(DataProviderRunner.class)
@@ -120,6 +122,20 @@ public class TelemetryDataJsonWriterTest {
""".formatted(data.getDefaultQualityGate()));
}

@Test
public void writes_sonarWay_qg() {
TelemetryData data = telemetryBuilder()
.setSonarWayQualityGate("sonarWayUUID")
.build();

String json = writeTelemetryData(data);
assertJson(json).isSimilarTo("""
{
"sonarway_quality_gate_uuid": "%s"
}
""".formatted(data.getSonarWayQualityGate()));
}

@Test
public void writes_database() {
String name = randomAlphabetic(12);
@@ -517,20 +533,56 @@ public class TelemetryDataJsonWriterTest {
assertJson(json).isSimilarTo("""
{
"quality-gates": [
{
"uuid": "uuid-0",
"caycStatus": "non-compliant"
},
{
"uuid": "uuid-1",
"caycStatus": "compliant"
},
{
"uuid": "uuid-2",
"caycStatus": "over-compliant"
}
]
}
{
"uuid": "uuid-0",
"caycStatus": "non-compliant",
"conditions": [
{
"metric": "new_coverage",
"comparison_operator": "LT",
"error_value": "80"
},
{
"metric": "new_duplicated_lines_density",
"comparison_operator": "GT",
"error_value": "3"
}
]
},
{
"uuid": "uuid-1",
"caycStatus": "compliant",
"conditions": [
{
"metric": "new_coverage",
"comparison_operator": "LT",
"error_value": "80"
},
{
"metric": "new_duplicated_lines_density",
"comparison_operator": "GT",
"error_value": "3"
}
]
},
{
"uuid": "uuid-2",
"caycStatus": "over-compliant",
"conditions": [
{
"metric": "new_coverage",
"comparison_operator": "LT",
"error_value": "80"
},
{
"metric": "new_duplicated_lines_density",
"comparison_operator": "GT",
"error_value": "3"
}
]
}
]
}
""");
}

@@ -693,9 +745,15 @@ public class TelemetryDataJsonWriterTest {
}

private List<TelemetryData.QualityGate> attachQualityGates() {
return List.of(new TelemetryData.QualityGate("uuid-0", "non-compliant"),
new TelemetryData.QualityGate("uuid-1", "compliant"),
new TelemetryData.QualityGate("uuid-2", "over-compliant"));
List<Condition> qualityGateConditions = attachQualityGateConditions();
return List.of(new TelemetryData.QualityGate("uuid-0", "non-compliant", qualityGateConditions),
new TelemetryData.QualityGate("uuid-1", "compliant", qualityGateConditions),
new TelemetryData.QualityGate("uuid-2", "over-compliant", qualityGateConditions));
}

private List<Condition> attachQualityGateConditions() {
return List.of(new Condition("new_coverage", fromDbValue("LT"), "80"),
new Condition("new_duplicated_lines_density", fromDbValue("GT"), "3"));
}

private List<TelemetryData.Branch> attachBranches() {

+ 17
- 5
server/sonar-webserver-core/src/it/java/org/sonar/server/telemetry/TelemetryDataLoaderImplIT.java View File

@@ -26,12 +26,14 @@ import java.sql.DatabaseMetaData;
import java.sql.SQLException;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.junit.Before;
import org.junit.Rule;
@@ -53,6 +55,7 @@ import org.sonar.db.metric.MetricDto;
import org.sonar.db.newcodeperiod.NewCodePeriodType;
import org.sonar.db.project.CreationMethod;
import org.sonar.db.property.PropertyDto;
import org.sonar.db.qualitygate.QualityGateConditionDto;
import org.sonar.db.qualitygate.QualityGateDto;
import org.sonar.db.qualityprofile.QProfileDto;
import org.sonar.db.user.UserDbTester;
@@ -137,7 +140,7 @@ public class TelemetryDataLoaderImplIT {

@Before
public void setUpBuiltInQualityGate() {
String builtInQgName = "Sonar Way";
String builtInQgName = "Sonar way";
builtInDefaultQualityGate = db.qualityGates().insertQualityGate(qg -> qg.setName(builtInQgName).setBuiltIn(true));
when(qualityGateCaycChecker.checkCaycCompliant(any(), any())).thenReturn(NON_COMPLIANT);
db.qualityGates().setDefaultQualityGate(builtInDefaultQualityGate);
@@ -219,6 +222,9 @@ public class TelemetryDataLoaderImplIT {
QualityGateDto qualityGate1 = db.qualityGates().insertQualityGate(qg -> qg.setName("QG1").setBuiltIn(true));
QualityGateDto qualityGate2 = db.qualityGates().insertQualityGate(qg -> qg.setName("QG2"));

QualityGateConditionDto condition1 = db.qualityGates().addCondition(qualityGate1, vulnerabilitiesDto, c -> c.setOperator("GT").setErrorThreshold("80"));
QualityGateConditionDto condition2 = db.qualityGates().addCondition(qualityGate2, securityHotspotsDto, c -> c.setOperator("LT").setErrorThreshold("2"));

// quality profiles
QProfileDto javaQP = db.qualityProfiles().insert(qProfileDto -> qProfileDto.setLanguage("java"));
QProfileDto kotlinQP = db.qualityProfiles().insert(qProfileDto -> qProfileDto.setLanguage("kotlin"));
@@ -245,6 +251,7 @@ public class TelemetryDataLoaderImplIT {
assertThat(data.getVersion()).isEqualTo(version);
assertThat(data.getEdition()).contains(DEVELOPER);
assertThat(data.getDefaultQualityGate()).isEqualTo(builtInDefaultQualityGate.getUuid());
assertThat(data.getSonarWayQualityGate()).isEqualTo(builtInDefaultQualityGate.getUuid());
assertThat(data.getNcdId()).isEqualTo(NewCodeDefinition.getInstanceDefault().hashCode());
assertThat(data.getMessageSequenceNumber()).isOne();
assertDatabaseMetadata(data.getDatabase());
@@ -294,11 +301,14 @@ public class TelemetryDataLoaderImplIT {
tuple("branch", NewCodePeriodType.REFERENCE_BRANCH.name(), branch1.uuid()));

assertThat(data.getQualityGates())
.extracting(TelemetryData.QualityGate::uuid, TelemetryData.QualityGate::caycStatus)
.extracting(TelemetryData.QualityGate::uuid, TelemetryData.QualityGate::caycStatus,
qg -> qg.conditions().stream()
.map(condition -> tuple(condition.getMetricKey(), condition.getOperator().getDbValue(), condition.getErrorThreshold(), condition.isOnLeakPeriod()))
.collect(Collectors.toList()))
.containsExactlyInAnyOrder(
tuple(builtInDefaultQualityGate.getUuid(), "non-compliant"),
tuple(qualityGate1.getUuid(), "non-compliant"),
tuple(qualityGate2.getUuid(), "non-compliant"));
tuple(builtInDefaultQualityGate.getUuid(), "non-compliant", Collections.emptyList()),
tuple(qualityGate1.getUuid(), "non-compliant", List.of(tuple(vulnerabilitiesDto.getKey(), condition1.getOperator(), condition1.getErrorThreshold(), false))),
tuple(qualityGate2.getUuid(), "non-compliant", List.of(tuple(securityHotspotsDto.getKey(), condition2.getOperator(), condition2.getErrorThreshold(), false))));

assertThat(data.getQualityProfiles())
.extracting(TelemetryData.QualityProfile::uuid, TelemetryData.QualityProfile::isBuiltIn)
@@ -702,6 +712,8 @@ public class TelemetryDataLoaderImplIT {
.setCreatedAt(1L));
}



@DataProvider
public static Set<String> getScimFeatureStatues() {
HashSet<String> result = new HashSet<>();

+ 42
- 2
server/sonar-webserver-core/src/main/java/org/sonar/server/telemetry/TelemetryDataLoaderImpl.java View File

@@ -57,10 +57,12 @@ import org.sonar.db.project.ProjectDto;
import org.sonar.db.property.PropertyDto;
import org.sonar.db.property.PropertyQuery;
import org.sonar.db.qualitygate.ProjectQgateAssociationDto;
import org.sonar.db.qualitygate.QualityGateConditionDto;
import org.sonar.db.qualitygate.QualityGateDto;
import org.sonar.server.management.ManagedInstanceService;
import org.sonar.server.platform.ContainerSupport;
import org.sonar.server.property.InternalProperties;
import org.sonar.server.qualitygate.Condition;
import org.sonar.server.qualitygate.QualityGateCaycChecker;
import org.sonar.server.qualitygate.QualityGateFinder;
import org.sonar.server.telemetry.TelemetryData.Database;
@@ -86,6 +88,7 @@ import static org.sonar.core.platform.EditionProvider.Edition.ENTERPRISE;
import static org.sonar.db.newcodeperiod.NewCodePeriodType.REFERENCE_BRANCH;
import static org.sonar.server.metric.UnanalyzedLanguageMetrics.UNANALYZED_CPP_KEY;
import static org.sonar.server.metric.UnanalyzedLanguageMetrics.UNANALYZED_C_KEY;
import static org.sonar.server.qualitygate.Condition.Operator.fromDbValue;
import static org.sonar.server.telemetry.TelemetryDaemon.I_PROP_MESSAGE_SEQUENCE;

@ServerSide
@@ -167,9 +170,11 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader {
data.setNewCodeDefinitions(newCodeDefinitions);

String defaultQualityGateUuid = qualityGateFinder.getDefault(dbSession).getUuid();
String sonarWayQualityGateUuid = qualityGateFinder.getSonarWay(dbSession).getUuid();
List<ProjectDto> projects = dbClient.projectDao().selectProjects(dbSession);

data.setDefaultQualityGate(defaultQualityGateUuid);
data.setSonarWayQualityGate(sonarWayQualityGateUuid);
resolveUnanalyzedLanguageCode(data, dbSession);
resolveProjectStatistics(data, dbSession, defaultQualityGateUuid, projects);
resolveProjects(data, dbSession);
@@ -217,7 +222,7 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader {
this.qualityProfileByProjectAndLanguage.clear();
}

private void loadNewCodeDefinitions(DbSession dbSession, List<BranchMeasuresDto> branchMeasuresDtos) {
private void loadNewCodeDefinitions(DbSession dbSession, List<BranchMeasuresDto> branchMeasuresDtos) {
var branchUuidByKey = branchMeasuresDtos.stream()
.collect(Collectors.toMap(dto -> createBranchUniqueKey(dto.getProjectUuid(), dto.getBranchKey()), BranchMeasuresDto::getBranchUuid));
List<NewCodePeriodDto> newCodePeriodDtos = dbClient.newCodePeriodDao().selectAll(dbSession);
@@ -380,15 +385,50 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader {
private void resolveQualityGates(TelemetryData.Builder data, DbSession dbSession) {
List<TelemetryData.QualityGate> qualityGates = new ArrayList<>();
Collection<QualityGateDto> qualityGateDtos = dbClient.qualityGateDao().selectAll(dbSession);
Collection<QualityGateConditionDto> qualityGateConditions = dbClient.gateConditionDao().selectAll(dbSession);
Map<String, MetricDto> metricsByUuid = getMetricsByUuid(dbSession, qualityGateConditions);

Map<String, List<Condition>> conditionsMap = mapQualityGateConditions(qualityGateConditions, metricsByUuid);

for (QualityGateDto qualityGateDto : qualityGateDtos) {
String qualityGateUuid = qualityGateDto.getUuid();
List<Condition> conditions = conditionsMap.getOrDefault(qualityGateUuid, Collections.emptyList());
qualityGates.add(
new TelemetryData.QualityGate(qualityGateDto.getUuid(), qualityGateCaycChecker.checkCaycCompliant(dbSession,
qualityGateDto.getUuid()).toString()));
qualityGateDto.getUuid()).toString(), conditions));
}

data.setQualityGates(qualityGates);
}

private static Map<String, List<Condition>> mapQualityGateConditions(Collection<QualityGateConditionDto> qualityGateConditions, Map<String, MetricDto> metricsByUuid) {
Map<String, List<Condition>> conditionsMap = new HashMap<>();

for (QualityGateConditionDto condition : qualityGateConditions) {
String qualityGateUuid = condition.getQualityGateUuid();

MetricDto metricDto = metricsByUuid.get(condition.getMetricUuid());
String metricKey = metricDto != null ? metricDto.getKey() : "Unknown Metric";

Condition telemetryCondition = new Condition(
metricKey,
fromDbValue(condition.getOperator()),
condition.getErrorThreshold()
);

conditionsMap
.computeIfAbsent(qualityGateUuid, k -> new ArrayList<>())
.add(telemetryCondition);
}

return conditionsMap;
}

private Map<String, MetricDto> getMetricsByUuid(DbSession dbSession, Collection<QualityGateConditionDto> conditions) {
Set<String> metricUuids = conditions.stream().map(QualityGateConditionDto::getMetricUuid).collect(Collectors.toSet());
return dbClient.metricDao().selectByUuids(dbSession, metricUuids).stream().filter(MetricDto::isEnabled).collect(Collectors.toMap(MetricDto::getUuid, Function.identity()));
}

private void resolveUsers(TelemetryData.Builder data, DbSession dbSession) {
data.setUsers(dbClient.userDao().selectUsersForTelemetry(dbSession));
}

+ 1
- 1
server/sonar-webserver-webapi/src/main/java/org/sonar/server/qualitygate/RegisterQualityGates.java View File

@@ -51,11 +51,11 @@ import static org.sonar.api.measures.CoreMetrics.NEW_SECURITY_HOTSPOTS_REVIEWED_
import static org.sonar.api.measures.CoreMetrics.NEW_VIOLATIONS_KEY;
import static org.sonar.db.qualitygate.QualityGateConditionDto.OPERATOR_GREATER_THAN;
import static org.sonar.db.qualitygate.QualityGateConditionDto.OPERATOR_LESS_THAN;
import static org.sonar.server.qualitygate.QualityGate.BUILTIN_QUALITY_GATE_NAME;

public class RegisterQualityGates implements Startable {

private static final Logger LOGGER = LoggerFactory.getLogger(RegisterQualityGates.class);
private static final String BUILTIN_QUALITY_GATE_NAME = "Sonar way";
private static final List<QualityGateCondition> QUALITY_GATE_CONDITIONS = asList(
new QualityGateCondition().setMetricKey(NEW_VIOLATIONS_KEY).setOperator(OPERATOR_GREATER_THAN).setErrorThreshold("0"),
new QualityGateCondition().setMetricKey(NEW_COVERAGE_KEY).setOperator(OPERATOR_LESS_THAN).setErrorThreshold("80"),

Loading…
Cancel
Save