diff options
author | Eric Giffon <eric.giffon@sonarsource.com> | 2024-12-19 16:27:23 +0100 |
---|---|---|
committer | sonartech <sonartech@sonarsource.com> | 2024-12-20 20:03:10 +0000 |
commit | 611b4ce3299d50021c3fd37b532f7ec46e85187c (patch) | |
tree | 46fc47f899ae77d73845da331a936f690505eeda | |
parent | 57439296944b342664d1ab1d42d0502ff28f261a (diff) | |
download | sonarqube-611b4ce3299d50021c3fd37b532f7ec46e85187c.tar.gz sonarqube-611b4ce3299d50021c3fd37b532f7ec46e85187c.zip |
SONAR-24057 Optimize querying of branch measures to only load required measures in memory
18 files changed, 138 insertions, 324 deletions
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ProjectNclocComputationStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ProjectNclocComputationStep.java index 2d2c4baae75..be29f232d2b 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ProjectNclocComputationStep.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ProjectNclocComputationStep.java @@ -19,10 +19,12 @@ */ package org.sonar.ce.task.projectanalysis.step; +import java.util.List; import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolder; import org.sonar.ce.task.step.ComputationStep; import org.sonar.db.DbClient; import org.sonar.db.DbSession; +import org.sonar.db.component.BranchDto; public class ProjectNclocComputationStep implements ComputationStep { @@ -38,7 +40,10 @@ public class ProjectNclocComputationStep implements ComputationStep { public void execute(Context context) { try (DbSession dbSession = dbClient.openSession(false)) { String projectUuid = analysisMetadataHolder.getProject().getUuid(); - long maxncloc = dbClient.measureDao().findNclocOfBiggestBranchForProject(dbSession, projectUuid); + List<String> branchUuids = dbClient.branchDao().selectByProjectUuid(dbSession, projectUuid).stream() + .map(BranchDto::getUuid) + .toList(); + long maxncloc = dbClient.measureDao().findNclocOfBiggestBranch(dbSession, branchUuids); dbClient.projectDao().updateNcloc(dbSession, projectUuid, maxncloc); dbSession.commit(); } diff --git a/server/sonar-db-dao/src/it/java/org/sonar/db/component/BranchDaoIT.java b/server/sonar-db-dao/src/it/java/org/sonar/db/component/BranchDaoIT.java index f5096c9a414..493f4d82195 100644 --- a/server/sonar-db-dao/src/it/java/org/sonar/db/component/BranchDaoIT.java +++ b/server/sonar-db-dao/src/it/java/org/sonar/db/component/BranchDaoIT.java @@ -52,6 +52,7 @@ import static org.assertj.core.api.Assertions.tuple; import static org.sonar.api.measures.CoreMetrics.ALERT_STATUS_KEY; import static org.sonar.db.component.BranchType.BRANCH; import static org.sonar.db.component.BranchType.PULL_REQUEST; +import static org.sonar.db.qualityprofile.QualityProfileTesting.newQualityProfileDto; class BranchDaoIT { @@ -871,6 +872,40 @@ class BranchDaoIT { assertThat(branchDtos).extracting(BranchDto::isMain).allMatch(b -> true); } + @Test + void selectMainBranches() { + ProjectData projectData1 = db.components().insertPrivateProject(); + BranchDto branch1 = projectData1.getMainBranchDto(); + db.components().insertProjectBranch(projectData1.getProjectDto()); + + ProjectData projectData2 = db.components().insertPrivateProject(); + BranchDto branch3 = projectData2.getMainBranchDto(); + + List<BranchDto> branchDtos = underTest.selectMainBranches(dbSession); + + assertThat(branchDtos).hasSize(2); + assertThat(branchDtos).extracting(BranchDto::getUuid, BranchDto::getProjectUuid, BranchDto::isMain).containsExactlyInAnyOrder( + tuple(branch1.getUuid(), projectData1.projectUuid(), true), + tuple(branch3.getUuid(), projectData2.projectUuid(), true)); + } + + @Test + void selectMainBranchesAssociatedToDefaultQualityProfile() { + ProjectData projectData1 = db.components().insertPrivateProject(); + BranchDto branch1 = projectData1.getMainBranchDto(); + db.components().insertProjectBranch(projectData1.getProjectDto()); + + ProjectData projectData2 = db.components().insertPrivateProject(); + + db.qualityProfiles().associateWithProject(projectData2.getProjectDto(), newQualityProfileDto()); + + List<BranchDto> branchDtos = underTest.selectMainBranchesAssociatedToDefaultQualityProfile(dbSession); + + assertThat(branchDtos).hasSize(1); + assertThat(branchDtos).extracting(BranchDto::getUuid, BranchDto::getProjectUuid, BranchDto::isMain).containsExactly( + tuple(branch1.getUuid(), projectData1.projectUuid(), true)); + } + private void insertBranchesForProjectUuids(boolean mainBranch, String... uuids) { for (String uuid : uuids) { BranchDto dto = new BranchDto(); diff --git a/server/sonar-db-dao/src/it/java/org/sonar/db/measure/MeasureDaoIT.java b/server/sonar-db-dao/src/it/java/org/sonar/db/measure/MeasureDaoIT.java index 86d14f5e0f8..6c466caa396 100644 --- a/server/sonar-db-dao/src/it/java/org/sonar/db/measure/MeasureDaoIT.java +++ b/server/sonar-db-dao/src/it/java/org/sonar/db/measure/MeasureDaoIT.java @@ -30,12 +30,12 @@ import org.apache.ibatis.session.ResultHandler; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.sonar.api.measures.CoreMetrics; -import org.sonar.db.component.ComponentQualifiers; import org.sonar.api.utils.System2; import org.sonar.db.DbSession; import org.sonar.db.DbTester; import org.sonar.db.component.BranchDto; import org.sonar.db.component.ComponentDto; +import org.sonar.db.component.ComponentQualifiers; import org.sonar.db.component.ProjectData; import org.sonar.db.metric.MetricDto; @@ -52,7 +52,6 @@ import static org.sonar.db.component.ComponentTesting.newDirectory; import static org.sonar.db.component.ComponentTesting.newFileDto; import static org.sonar.db.component.ComponentTesting.newPrivateProjectDto; import static org.sonar.db.measure.MeasureTesting.newMeasure; -import static org.sonar.db.qualityprofile.QualityProfileTesting.newQualityProfileDto; class MeasureDaoIT { @@ -306,28 +305,6 @@ class MeasureDaoIT { } @Test - void scrollSelectByComponentUuid() { - List<MeasureDto> results = new ArrayList<>(); - MetricDto metric = db.measures().insertMetric(); - MetricDto metric2 = db.measures().insertMetric(); - ComponentDto project = db.components().insertPrivateProject().getMainBranchComponent(); - ComponentDto project2 = db.components().insertPrivateProject().getMainBranchComponent(); - underTest.insertOrUpdate(db.getSession(), newMeasure(project, metric, 3.14)); - underTest.insertOrUpdate(db.getSession(), newMeasure(project, metric2, 4.54)); - underTest.insertOrUpdate(db.getSession(), newMeasure(project2, metric, 99.99)); - underTest.scrollSelectByComponentUuid(db.getSession(), project.uuid(), context -> results.add(context.getResultObject())); - - assertThat(results).hasSize(1); - assertThat(results).flatExtracting(m -> m.getMetricValues().entrySet().stream() - .map(entry -> tuple(m.getComponentUuid(), entry.getKey(), entry.getValue())) - .toList()) - .containsExactlyInAnyOrder( - tuple(project.uuid(), metric.getKey(), 3.14), - tuple(project.uuid(), metric2.getKey(), 4.54) - ); - } - - @Test void select_measure_hashes_for_branch() { MeasureDto measure1 = new MeasureDto() .setComponentUuid("c1") @@ -354,39 +331,6 @@ class MeasureDaoIT { } @Test - void select_branch_measures_for_project() { - - // 2 branches on the same project, 1 branch on another project - ProjectData projectData = db.components().insertPrivateProject(); - BranchDto branch1 = projectData.getMainBranchDto(); - BranchDto branch2 = db.components().insertProjectBranch(projectData.getProjectDto()); - BranchDto branch3 = db.components().insertPrivateProject().getMainBranchDto(); - - // Insert measures for each branch and for a random component on branch1 - MetricDto metric = db.measures().insertMetric(); - MeasureDto measure1 = newMeasure(branch1, metric, 3); - MeasureDto measure2 = newMeasure(branch2, metric, 4); - MeasureDto measure3 = newMeasure(branch3, metric, 5); - MeasureDto measure4 = newMeasure(db.components().insertFile(branch1), metric, 6); - - underTest.insertOrUpdate(db.getSession(), measure1); - underTest.insertOrUpdate(db.getSession(), measure2); - underTest.insertOrUpdate(db.getSession(), measure3); - underTest.insertOrUpdate(db.getSession(), measure4); - - List<MeasureDto> measures = underTest.selectBranchMeasuresForProject(db.getSession(), projectData.projectUuid()); - assertThat(measures).hasSize(2); - assertThat(measures) - .flatExtracting(m -> m.getMetricValues().entrySet().stream() - .map(entry -> tuple(m.getComponentUuid(), m.getBranchUuid(), entry.getKey(), entry.getValue())) - .toList()) - .containsExactlyInAnyOrder( - tuple(branch1.getUuid(), branch1.getUuid(), metric.getKey(), 3.0), - tuple(branch2.getUuid(), branch2.getUuid(), metric.getKey(), 4.0) - ); - } - - @Test void selectTreeByQuery_return_leaves_and_base_component() { List<MeasureDto> results = new ArrayList<>(); MetricDto metric1 = db.measures().insertMetric(); @@ -533,135 +477,23 @@ class MeasureDaoIT { } @Test - void selectAllForProjectMainBranches() { - ProjectData projectData1 = db.components().insertPrivateProject(); - BranchDto branch1 = projectData1.getMainBranchDto(); - BranchDto branch2 = db.components().insertProjectBranch(projectData1.getProjectDto()); - - ProjectData projectData2 = db.components().insertPrivateProject(); - BranchDto branch3 = projectData2.getMainBranchDto(); - BranchDto branch4 = db.components().insertProjectBranch(projectData2.getProjectDto()); - - // Insert measures for each branch and for a random component on branch1 - MetricDto metric = db.measures().insertMetric(); - MeasureDto measure1 = newMeasure(branch1, metric, 3); - MeasureDto measure2 = newMeasure(branch2, metric, 4); - MeasureDto measure3 = newMeasure(branch3, metric, 5); - MeasureDto measure4 = newMeasure(branch4, metric, 6); - MeasureDto measure5 = newMeasure(db.components().insertFile(branch1), metric, 7); - - underTest.insertOrUpdate(db.getSession(), measure1); - underTest.insertOrUpdate(db.getSession(), measure2); - underTest.insertOrUpdate(db.getSession(), measure3); - underTest.insertOrUpdate(db.getSession(), measure4); - underTest.insertOrUpdate(db.getSession(), measure5); - - List<ProjectMainBranchMeasureDto> measures = underTest.selectAllForProjectMainBranches(db.getSession()); - assertThat(measures).hasSize(2); - assertThat(measures) - .flatExtracting(m -> m.getMetricValues().entrySet().stream() - .map(entry -> tuple(m.getProjectUuid(), entry.getKey(), entry.getValue())) - .toList()) - .containsExactlyInAnyOrder( - tuple(projectData1.projectUuid(), metric.getKey(), 3.0), - tuple(projectData2.projectUuid(), metric.getKey(), 5.0) - ); - } - - @Test - void selectAllForProjectMainBranchesAssociatedToDefaultQualityProfile() { - ProjectData projectData1 = db.components().insertPrivateProject(); - BranchDto branch1 = projectData1.getMainBranchDto(); - BranchDto branch2 = db.components().insertProjectBranch(projectData1.getProjectDto()); - - ProjectData projectData2 = db.components().insertPrivateProject(); - BranchDto branch3 = projectData2.getMainBranchDto(); - - // Insert measures for each branch and for a random component on branch1 - MetricDto metric = db.measures().insertMetric(); - MeasureDto measure1 = newMeasure(branch1, metric, 3); - MeasureDto measure2 = newMeasure(branch2, metric, 4); - MeasureDto measure3 = newMeasure(branch3, metric, 5); - MeasureDto measure4 = newMeasure(db.components().insertFile(branch1), metric, 7); - - underTest.insertOrUpdate(db.getSession(), measure1); - underTest.insertOrUpdate(db.getSession(), measure2); - underTest.insertOrUpdate(db.getSession(), measure3); - underTest.insertOrUpdate(db.getSession(), measure4); - - db.qualityProfiles().associateWithProject(projectData2.getProjectDto(), newQualityProfileDto()); - - List<ProjectMainBranchMeasureDto> measures = - underTest.selectAllForProjectMainBranchesAssociatedToDefaultQualityProfile(db.getSession()); - assertThat(measures).hasSize(1); - assertThat(measures) - .flatExtracting(m -> m.getMetricValues().entrySet().stream() - .map(entry -> tuple(m.getProjectUuid(), entry.getKey(), entry.getValue())) - .toList()) - .containsExactly( - tuple(projectData1.projectUuid(), metric.getKey(), 3.0) - ); - } - - @Test - void selectAllForMainBranches() { - ProjectData projectData1 = db.components().insertPrivateProject(); - BranchDto branch1 = projectData1.getMainBranchDto(); - BranchDto branch2 = db.components().insertProjectBranch(projectData1.getProjectDto()); - - ProjectData projectData2 = db.components().insertPrivateProject(); - BranchDto branch3 = projectData2.getMainBranchDto(); - BranchDto branch4 = db.components().insertProjectBranch(projectData2.getProjectDto()); - - // Insert measures for each branch and for a random component on branch1 - MetricDto metric = db.measures().insertMetric(); - MeasureDto measure1 = newMeasure(branch1, metric, 3); - MeasureDto measure2 = newMeasure(branch2, metric, 4); - MeasureDto measure3 = newMeasure(branch3, metric, 5); - MeasureDto measure4 = newMeasure(branch4, metric, 6); - MeasureDto measure5 = newMeasure(db.components().insertFile(branch1), metric, 7); - - underTest.insertOrUpdate(db.getSession(), measure1); - underTest.insertOrUpdate(db.getSession(), measure2); - underTest.insertOrUpdate(db.getSession(), measure3); - underTest.insertOrUpdate(db.getSession(), measure4); - underTest.insertOrUpdate(db.getSession(), measure5); - - List<MeasureDto> measures = underTest.selectAllForMainBranches(db.getSession()); - assertThat(measures).hasSize(2); - assertThat(measures) - .flatExtracting(m -> m.getMetricValues().entrySet().stream() - .map(entry -> tuple(m.getComponentUuid(), m.getBranchUuid(), entry.getKey(), entry.getValue())) - .toList()) - .containsExactlyInAnyOrder( - tuple(branch1.getUuid(), branch1.getUuid(), metric.getKey(), 3.0), - tuple(branch3.getUuid(), branch3.getUuid(), metric.getKey(), 5.0) - ); - } - - @Test - void findNclocOfBiggestBranchForProject() { - // 2 branches on the same project, 1 branch on 2 other projects + void findNclocOfBiggestBranch() { ProjectData projectData = db.components().insertPrivateProject(); BranchDto branch1 = projectData.getMainBranchDto(); BranchDto branch2 = db.components().insertProjectBranch(projectData.getProjectDto()); - BranchDto branch3 = db.components().insertPrivateProject().getMainBranchDto(); - ProjectData project2 = db.components().insertPrivateProject(); // Insert measures for each branch and for a random component on branch1 MetricDto metric = db.measures().insertMetric(metricDto -> metricDto.setKey(CoreMetrics.NCLOC_KEY)); MeasureDto measure1 = newMeasure(branch1, metric, 3); MeasureDto measure2 = newMeasure(branch2, metric, 4); - MeasureDto measure3 = newMeasure(branch3, metric, 5); - MeasureDto measure4 = newMeasure(db.components().insertFile(branch1), metric, 6); + MeasureDto measure3 = newMeasure(db.components().insertFile(branch1), metric, 6); underTest.insertOrUpdate(db.getSession(), measure1); underTest.insertOrUpdate(db.getSession(), measure2); underTest.insertOrUpdate(db.getSession(), measure3); - underTest.insertOrUpdate(db.getSession(), measure4); - assertThat(underTest.findNclocOfBiggestBranchForProject(db.getSession(), projectData.projectUuid())).isEqualTo(4); - assertThat(underTest.findNclocOfBiggestBranchForProject(db.getSession(), project2.projectUuid())).isZero(); + assertThat(underTest.findNclocOfBiggestBranch(db.getSession(), List.of(branch1.getUuid(), branch2.getUuid()))) + .isEqualTo(4); } private MeasureDto newMeasureForMetrics(ComponentDto componentDto, MetricDto... metrics) { diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java b/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java index 84223ad785c..bdbccd5491e 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java @@ -91,7 +91,6 @@ import org.sonar.db.issue.NewCodeReferenceIssueDto; import org.sonar.db.issue.PrIssueDto; import org.sonar.db.measure.LargestBranchNclocDto; import org.sonar.db.measure.MeasureMapper; -import org.sonar.db.measure.ProjectLocDistributionDto; import org.sonar.db.measure.ProjectMeasureDto; import org.sonar.db.measure.ProjectMeasureMapper; import org.sonar.db.metric.MetricMapper; @@ -256,7 +255,6 @@ public class MyBatis { confBuilder.loadAlias("AnalysisPropertyValuePerProject", AnalysisPropertyValuePerProject.class); confBuilder.loadAlias("ProjectAlmKeyAndProject", ProjectAlmKeyAndProject.class); confBuilder.loadAlias("PrAndBranchCountByProjectDto", PrBranchAnalyzedLanguageCountByProjectDto.class); - confBuilder.loadAlias("ProjectLocDistribution", ProjectLocDistributionDto.class); confBuilder.loadAlias("PurgeableAnalysis", PurgeableAnalysisDto.class); confBuilder.loadAlias("PushEvent", PushEventDto.class); confBuilder.loadAlias("QualityGateCondition", QualityGateConditionDto.class); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchDao.java index 0e7eb4bbb3c..dd06c36bebf 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchDao.java @@ -206,4 +206,12 @@ public class BranchDao implements Dao { long yesterday = ZonedDateTime.now(ZoneId.systemDefault()).minusDays(1).toInstant().toEpochMilli(); return mapper(dbSession).selectBranchMeasuresWithCaycMetric(yesterday); } + + public List<BranchDto> selectMainBranches(DbSession dbSession) { + return mapper(dbSession).selectMainBranches(); + } + + public List<BranchDto> selectMainBranchesAssociatedToDefaultQualityProfile(DbSession dbSession) { + return mapper(dbSession).selectMainBranchesAssociatedToDefaultQualityProfile(); + } } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchMapper.java index 864e2b61642..ae4dde26dae 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchMapper.java @@ -77,4 +77,8 @@ public interface BranchMapper { List<BranchDto> selectMainBranchesByProjectUuids(@Param("projectUuids") Collection<String> projectUuids); List<BranchMeasuresDto> selectBranchMeasuresWithCaycMetric(long yesterday); + + List<BranchDto> selectMainBranches(); + + List<BranchDto> selectMainBranchesAssociatedToDefaultQualityProfile(); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureDao.java index ab619fcb46f..7777008ae45 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureDao.java @@ -19,6 +19,7 @@ */ package org.sonar.db.measure; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -32,7 +33,7 @@ import org.sonar.db.component.ComponentDto; import static java.util.Collections.singletonList; import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY; -import static org.sonar.db.DatabaseUtils.executeLargeInputs; +import static org.sonar.db.DatabaseUtils.executeLargeInputsWithoutOutput; public class MeasureDao implements Dao { @@ -91,35 +92,33 @@ public class MeasureDao implements Dao { return Optional.empty(); } + /** + * Get the measures of the specified components and metrics. + * This method will scroll through the results and remove the non-requested metrics, to avoid loading all the measures in memory. + */ public List<MeasureDto> selectByComponentUuidsAndMetricKeys(DbSession dbSession, Collection<String> largeComponentUuids, Collection<String> metricKeys) { if (largeComponentUuids.isEmpty() || metricKeys.isEmpty()) { return Collections.emptyList(); } - return executeLargeInputs( - largeComponentUuids, - componentUuids -> mapper(dbSession).selectByComponentUuids(componentUuids)).stream() - .map(measureDto -> { + List<MeasureDto> result = new ArrayList<>(); + executeLargeInputsWithoutOutput(largeComponentUuids, + componentUuids -> mapper(dbSession).scrollSelectByComponentUuids(componentUuids, resultContext -> { + MeasureDto measureDto = resultContext.getResultObject(); measureDto.getMetricValues().entrySet().removeIf(entry -> !metricKeys.contains(entry.getKey())); - return measureDto; - }) - .filter(measureDto -> !measureDto.getMetricValues().isEmpty()) - .toList(); - } + if (!measureDto.getMetricValues().isEmpty()) { + result.add(measureDto); + } + })); - public void scrollSelectByComponentUuid(DbSession dbSession, String componentUuid, ResultHandler<MeasureDto> handler) { - mapper(dbSession).scrollSelectByComponentUuid(componentUuid, handler); + return result; } public Set<MeasureHash> selectMeasureHashesForBranch(DbSession dbSession, String branchUuid) { return mapper(dbSession).selectMeasureHashesForBranch(branchUuid); } - public List<MeasureDto> selectBranchMeasuresForProject(DbSession dbSession, String projectUuid) { - return mapper(dbSession).selectBranchMeasuresForProject(projectUuid); - } - public void selectTreeByQuery(DbSession dbSession, ComponentDto baseComponent, MeasureTreeQuery query, ResultHandler<MeasureDto> resultHandler) { if (query.returnsEmpty()) { @@ -128,20 +127,8 @@ public class MeasureDao implements Dao { mapper(dbSession).selectTreeByQuery(query, baseComponent.uuid(), query.getUuidPath(baseComponent), resultHandler); } - public List<ProjectMainBranchMeasureDto> selectAllForProjectMainBranches(DbSession dbSession) { - return mapper(dbSession).selectAllForProjectMainBranches(); - } - - public List<ProjectMainBranchMeasureDto> selectAllForProjectMainBranchesAssociatedToDefaultQualityProfile(DbSession dbSession) { - return mapper(dbSession).selectAllForProjectMainBranchesAssociatedToDefaultQualityProfile(); - } - - public List<MeasureDto> selectAllForMainBranches(DbSession dbSession) { - return mapper(dbSession).selectAllForMainBranches(); - } - - public long findNclocOfBiggestBranchForProject(DbSession dbSession, String projectUuid) { - List<MeasureDto> branchMeasures = mapper(dbSession).selectBranchMeasuresForProject(projectUuid); + public long findNclocOfBiggestBranch(DbSession dbSession, Collection<String> branchUuids) { + List<MeasureDto> branchMeasures = selectByComponentUuidsAndMetricKeys(dbSession, branchUuids, List.of(NCLOC_KEY)); long maxncloc = 0; for (MeasureDto measure : branchMeasures) { diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureMapper.java index b6baf817e2a..3e7c19441f1 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureMapper.java @@ -37,21 +37,13 @@ public interface MeasureMapper { List<MeasureDto> selectByComponentUuids(@Param("componentUuids") Collection<String> componentUuids); - void scrollSelectByComponentUuid(@Param("componentUuid") String componentUuid, ResultHandler<MeasureDto> handler); + void scrollSelectByComponentUuids(@Param("componentUuids") Collection<String> componentUuids, ResultHandler<MeasureDto> handler); Set<MeasureHash> selectMeasureHashesForBranch(@Param("branchUuid") String branchUuid); - List<MeasureDto> selectBranchMeasuresForProject(@Param("projectUuid") String projectUuid); - void selectTreeByQuery( @Param("query") MeasureTreeQuery measureQuery, @Param("baseUuid") String baseUuid, @Param("baseUuidPath") String baseUuidPath, ResultHandler<MeasureDto> resultHandler); - - List<ProjectMainBranchMeasureDto> selectAllForProjectMainBranches(); - - List<ProjectMainBranchMeasureDto> selectAllForProjectMainBranchesAssociatedToDefaultQualityProfile(); - - List<MeasureDto> selectAllForMainBranches(); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/ProjectMainBranchMeasureDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/ProjectMainBranchMeasureDto.java deleted file mode 100644 index 22aa5db4d6d..00000000000 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/ProjectMainBranchMeasureDto.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2024 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.db.measure; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import java.util.Map; -import java.util.TreeMap; - -public class ProjectMainBranchMeasureDto { - - private static final Gson GSON = new Gson(); - - private String projectUuid; - private Map<String, Object> metricValues = new TreeMap<>(); - - public ProjectMainBranchMeasureDto() { - // empty constructor - } - - public String getProjectUuid() { - return projectUuid; - } - - public Map<String, Object> getMetricValues() { - return metricValues; - } - - // used by MyBatis mapper - public void setJsonValue(String jsonValue) { - metricValues = GSON.fromJson(jsonValue, new TypeToken<TreeMap<String, Object>>() { - }.getType()); - } -} diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/component/BranchMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/component/BranchMapper.xml index b2b3993d2bf..f96a116d7c1 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/component/BranchMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/component/BranchMapper.xml @@ -304,4 +304,22 @@ from dual </select> + <select id="selectMainBranches" resultType="org.sonar.db.component.BranchDto"> + select <include refid="columns"/> + from project_branches pb + inner join projects p on p.uuid = pb.project_uuid + where + p.qualifier = 'TRK' + and pb.is_main = ${_true} + </select> + + <select id="selectMainBranchesAssociatedToDefaultQualityProfile" resultType="org.sonar.db.component.BranchDto"> + select <include refid="columns"/> + from project_branches pb + inner join projects p on p.uuid = pb.project_uuid + where + pb.is_main = ${_true} + and p.uuid not in (select project_uuid from project_qprofiles) + </select> + </mapper> diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/MeasureMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/MeasureMapper.xml index 118ef27e034..d014a3c4a67 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/MeasureMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/MeasureMapper.xml @@ -47,13 +47,16 @@ </foreach> </select> - <select id="scrollSelectByComponentUuid" resultType="org.sonar.db.measure.MeasureDto" fetchSize="${_scrollFetchSize}" + <select id="scrollSelectByComponentUuids" parameterType="map" resultType="org.sonar.db.measure.MeasureDto" fetchSize="10" resultSetType="FORWARD_ONLY"> select <include refid="columns"/> from measures m where - m.component_uuid = #{componentUuid, jdbcType=VARCHAR} + m.component_uuid in + <foreach item="componentUuid" collection="componentUuids" open="(" separator="," close=")"> + #{componentUuid, jdbcType=VARCHAR} + </foreach> </select> <select id="selectMeasureHashesForBranch" resultType="org.sonar.db.measure.MeasureHash"> @@ -62,15 +65,6 @@ where branch_uuid = #{branchUuid, jdbcType=VARCHAR} </select> - <select id="selectBranchMeasuresForProject" parameterType="map" resultType="org.sonar.db.measure.MeasureDto"> - select - <include refid="columns"/> - from measures m - inner join project_branches b on b.uuid = m.component_uuid - inner join projects p on p.uuid = b.project_uuid and p.qualifier = 'TRK' - where b.project_uuid = #{projectUuid,jdbcType=VARCHAR} - </select> - <select id="selectTreeByQuery" parameterType="map" resultType="org.sonar.db.measure.MeasureDto" fetchSize="${_scrollFetchSize}" resultSetType="FORWARD_ONLY"> select <include refid="columns"/> from measures m @@ -116,30 +110,4 @@ </where> </select> - <select id="selectAllForProjectMainBranches" resultType="org.sonar.db.measure.ProjectMainBranchMeasureDto"> - select p.uuid as projectUuid, m.json_value as jsonValue - from measures m - inner join project_branches pb on pb.uuid = m.component_uuid - inner join projects p on p.uuid = pb.project_uuid - where - p.qualifier = 'TRK' - and pb.is_main = ${_true} - </select> - - <select id="selectAllForProjectMainBranchesAssociatedToDefaultQualityProfile" resultType="org.sonar.db.measure.ProjectMainBranchMeasureDto"> - select p.uuid as projectUuid, m.json_value as jsonValue - from measures m - inner join project_branches pb on pb.uuid = m.component_uuid - inner join projects p on p.uuid = pb.project_uuid - where - pb.is_main = ${_true} - and p.uuid not in (select project_uuid from project_qprofiles) - </select> - - <select id="selectAllForMainBranches" resultType="org.sonar.db.measure.MeasureDto"> - select <include refid="columns"/> from measures m - inner join project_branches pb on pb.uuid = m.component_uuid and pb.is_main = ${_true} - inner join projects p on p.uuid = pb.project_uuid and p.qualifier = 'TRK' - </select> - </mapper> diff --git a/server/sonar-telemetry/src/main/java/org/sonar/telemetry/legacy/ProjectLocDistributionDataProvider.java b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/legacy/ProjectLocDistributionDataProvider.java index 03e01d9a951..5e0c9468ff4 100644 --- a/server/sonar-telemetry/src/main/java/org/sonar/telemetry/legacy/ProjectLocDistributionDataProvider.java +++ b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/legacy/ProjectLocDistributionDataProvider.java @@ -23,8 +23,8 @@ import java.util.ArrayList; import java.util.List; import org.sonar.db.DbClient; import org.sonar.db.DbSession; +import org.sonar.db.component.BranchDto; import org.sonar.db.measure.MeasureDto; -import org.sonar.db.measure.ProjectLocDistributionDto; import org.sonar.db.project.ProjectDto; import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY; @@ -45,7 +45,11 @@ public class ProjectLocDistributionDataProvider { List<ProjectLocDistributionDto> branchesWithLargestNcloc = new ArrayList<>(); List<ProjectDto> projects = dbClient.projectDao().selectProjects(dbSession); for (ProjectDto project : projects) { - List<MeasureDto> branchMeasures = dbClient.measureDao().selectBranchMeasuresForProject(dbSession, project.getUuid()); + List<String> branchUuids = dbClient.branchDao().selectByProjectUuid(dbSession, project.getUuid()).stream() + .map(BranchDto::getUuid) + .toList(); + List<MeasureDto> branchMeasures = dbClient.measureDao() + .selectByComponentUuidsAndMetricKeys(dbSession, branchUuids, List.of(NCLOC_KEY, NCLOC_LANGUAGE_DISTRIBUTION_KEY)); long maxncloc = 0; String largestBranchUuid = null; diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/ProjectLocDistributionDto.java b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/legacy/ProjectLocDistributionDto.java index 0f091369a00..6641b79bffc 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/ProjectLocDistributionDto.java +++ b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/legacy/ProjectLocDistributionDto.java @@ -17,7 +17,7 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.db.measure; +package org.sonar.telemetry.legacy; /** * Loc distribution per language for the largest branch in a project. diff --git a/server/sonar-telemetry/src/main/java/org/sonar/telemetry/legacy/TelemetryDataLoaderImpl.java b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/legacy/TelemetryDataLoaderImpl.java index b255a0b0c37..00326c1f883 100644 --- a/server/sonar-telemetry/src/main/java/org/sonar/telemetry/legacy/TelemetryDataLoaderImpl.java +++ b/server/sonar-telemetry/src/main/java/org/sonar/telemetry/legacy/TelemetryDataLoaderImpl.java @@ -46,12 +46,11 @@ import org.sonar.db.DbSession; import org.sonar.db.alm.setting.ALM; import org.sonar.db.alm.setting.ProjectAlmKeyAndProject; import org.sonar.db.component.AnalysisPropertyValuePerProject; +import org.sonar.db.component.BranchDto; import org.sonar.db.component.BranchMeasuresDto; import org.sonar.db.component.PrBranchAnalyzedLanguageCountByProjectDto; import org.sonar.db.component.SnapshotDto; import org.sonar.db.measure.MeasureDto; -import org.sonar.db.measure.ProjectLocDistributionDto; -import org.sonar.db.measure.ProjectMainBranchMeasureDto; import org.sonar.db.metric.MetricDto; import org.sonar.db.newcodeperiod.NewCodePeriodDto; import org.sonar.db.project.ProjectDto; @@ -281,7 +280,10 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { editionProvider.get() .filter(edition -> edition.equals(COMMUNITY)) .ifPresent(edition -> { - List<MeasureDto> measureDtos = dbClient.measureDao().selectAllForMainBranches(dbSession); + List<BranchDto> mainBranches = dbClient.branchDao().selectMainBranches(dbSession); + List<MeasureDto> measureDtos = dbClient.measureDao().selectByComponentUuidsAndMetricKeys(dbSession, + mainBranches.stream().map(BranchDto::getUuid).toList(), List.of(UNANALYZED_C_KEY, UNANALYZED_CPP_KEY)); + long numberOfUnanalyzedCMeasures = countProjectsHavingMeasure(measureDtos, UNANALYZED_C_KEY); long numberOfUnanalyzedCppMeasures = countProjectsHavingMeasure(measureDtos, UNANALYZED_CPP_KEY); @@ -488,13 +490,17 @@ public class TelemetryDataLoaderImpl implements TelemetryDataLoader { private Map<String, Map<String, Number>> getProjectMetricsByMetricKeys(DbSession dbSession, List<String> metricKeys) { Map<String, Map<String, Number>> measuresByProject = new HashMap<>(); - List<ProjectMainBranchMeasureDto> projectMainBranchMeasureDtos = dbClient.measureDao().selectAllForProjectMainBranches(dbSession); - for (ProjectMainBranchMeasureDto projectMainBranchMeasureDto : projectMainBranchMeasureDtos) { - Map<String, Number> measures = projectMainBranchMeasureDto.getMetricValues().entrySet().stream() - .filter(e -> metricKeys.contains(e.getKey())) + List<BranchDto> mainBranches = dbClient.branchDao().selectMainBranches(dbSession); + Map<String, String> branchUuidToProjectUuid = mainBranches.stream().collect(Collectors.toMap(BranchDto::getUuid, + BranchDto::getProjectUuid)); + List<MeasureDto> measureDtos = dbClient.measureDao().selectByComponentUuidsAndMetricKeys(dbSession, branchUuidToProjectUuid.keySet(), + metricKeys); + + for (MeasureDto measureDto : measureDtos) { + Map<String, Number> measures = measureDto.getMetricValues().entrySet().stream() .collect(toMap(Map.Entry::getKey, e -> Double.parseDouble(e.getValue().toString()))); - measuresByProject.put(projectMainBranchMeasureDto.getProjectUuid(), measures); + measuresByProject.put(branchUuidToProjectUuid.get(measureDto.getComponentUuid()), measures); } return measuresByProject; diff --git a/server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/qualityprofile/QualityProfileChangeEventServiceImpl.java b/server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/qualityprofile/QualityProfileChangeEventServiceImpl.java index 49ef88e5f0b..6610ae0a67f 100644 --- a/server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/qualityprofile/QualityProfileChangeEventServiceImpl.java +++ b/server/sonar-webserver-pushapi/src/main/java/org/sonar/server/pushapi/qualityprofile/QualityProfileChangeEventServiceImpl.java @@ -43,7 +43,8 @@ import org.sonar.core.util.rule.RuleChange; import org.sonar.core.util.rule.RuleSetChangedEvent; import org.sonar.db.DbClient; import org.sonar.db.DbSession; -import org.sonar.db.measure.ProjectMainBranchMeasureDto; +import org.sonar.db.component.BranchDto; +import org.sonar.db.measure.MeasureDto; import org.sonar.db.project.ProjectDto; import org.sonar.db.pushevent.PushEventDto; import org.sonar.db.qualityprofile.ActiveRuleDto; @@ -280,11 +281,16 @@ public class QualityProfileChangeEventServiceImpl implements QualityProfileChang private List<ProjectDto> getDefaultQualityProfileAssociatedProjects(DbSession dbSession, String language) { Set<String> associatedProjectUuids = new HashSet<>(); - List<ProjectMainBranchMeasureDto> measureDtos = dbClient.measureDao().selectAllForProjectMainBranchesAssociatedToDefaultQualityProfile(dbSession); - for (ProjectMainBranchMeasureDto measureDto : measureDtos) { - String distribution = (String) measureDto.getMetricValues().get(CoreMetrics.NCLOC_LANGUAGE_DISTRIBUTION_KEY); + List<BranchDto> branchDtos = dbClient.branchDao().selectMainBranchesAssociatedToDefaultQualityProfile(dbSession); + Map<String, String> branchUuidToProjectUuid = branchDtos.stream().collect(Collectors.toMap(BranchDto::getUuid, + BranchDto::getProjectUuid)); + List<MeasureDto> measureDtos = dbClient.measureDao().selectByComponentUuidsAndMetricKeys(dbSession, branchUuidToProjectUuid.keySet(), + List.of(CoreMetrics.NCLOC_LANGUAGE_DISTRIBUTION_KEY)); + + for (MeasureDto measureDto : measureDtos) { + String distribution = measureDto.getString(CoreMetrics.NCLOC_LANGUAGE_DISTRIBUTION_KEY); if (distribution != null && distributionContainsLanguage(distribution, language)) { - associatedProjectUuids.add(measureDto.getProjectUuid()); + associatedProjectUuids.add(branchUuidToProjectUuid.get(measureDto.getComponentUuid())); } } diff --git a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/component/ComponentCleanerService.java b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/component/ComponentCleanerService.java index c77132ec2a8..7d877aaacbc 100644 --- a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/component/ComponentCleanerService.java +++ b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/component/ComponentCleanerService.java @@ -20,11 +20,11 @@ package org.sonar.server.component; import java.util.List; -import org.sonar.db.component.ComponentQualifiers; import org.sonar.api.server.ServerSide; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.component.BranchDto; +import org.sonar.db.component.ComponentQualifiers; import org.sonar.db.entity.EntityDto; import org.sonar.db.project.ProjectDto; import org.sonar.server.es.Indexers; @@ -61,7 +61,10 @@ public class ComponentCleanerService { } private void updateProjectNcloc(DbSession dbSession, String projectUuid) { - long maxncloc = dbClient.measureDao().findNclocOfBiggestBranchForProject(dbSession, projectUuid); + List<String> branchUuids = dbClient.branchDao().selectByProjectUuid(dbSession, projectUuid).stream() + .map(BranchDto::getUuid) + .toList(); + long maxncloc = dbClient.measureDao().findNclocOfBiggestBranch(dbSession, branchUuids); dbClient.projectDao().updateNcloc(dbSession, projectUuid, maxncloc); } diff --git a/server/sonar-webserver/src/main/java/org/sonar/server/platform/telemetry/TelemetryNclocProvider.java b/server/sonar-webserver/src/main/java/org/sonar/server/platform/telemetry/TelemetryNclocProvider.java index 332312d85f3..1a0321bc7ec 100644 --- a/server/sonar-webserver/src/main/java/org/sonar/server/platform/telemetry/TelemetryNclocProvider.java +++ b/server/sonar-webserver/src/main/java/org/sonar/server/platform/telemetry/TelemetryNclocProvider.java @@ -25,7 +25,7 @@ import java.util.List; import java.util.Map; import org.sonar.db.DbClient; import org.sonar.db.DbSession; -import org.sonar.db.measure.ProjectLocDistributionDto; +import org.sonar.telemetry.legacy.ProjectLocDistributionDto; import org.sonar.telemetry.core.AbstractTelemetryDataProvider; import org.sonar.telemetry.core.Dimension; import org.sonar.telemetry.core.Granularity; diff --git a/server/sonar-webserver/src/test/java/org/sonar/server/platform/telemetry/TelemetryNclocProviderTest.java b/server/sonar-webserver/src/test/java/org/sonar/server/platform/telemetry/TelemetryNclocProviderTest.java index 4273dbdbe0f..8d0635fb6a3 100644 --- a/server/sonar-webserver/src/test/java/org/sonar/server/platform/telemetry/TelemetryNclocProviderTest.java +++ b/server/sonar-webserver/src/test/java/org/sonar/server/platform/telemetry/TelemetryNclocProviderTest.java @@ -24,7 +24,7 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.sonar.db.DbClient; import org.sonar.db.DbSession; -import org.sonar.db.measure.ProjectLocDistributionDto; +import org.sonar.telemetry.legacy.ProjectLocDistributionDto; import org.sonar.telemetry.core.Dimension; import org.sonar.telemetry.core.Granularity; import org.sonar.telemetry.legacy.ProjectLocDistributionDataProvider; @@ -64,4 +64,3 @@ class TelemetryNclocProviderTest { assertThat(telemetryNclocProvider.getValues()).containsEntry("js", 1500L); } } - |