From: Pierre Date: Tue, 28 Mar 2023 08:16:41 +0000 (+0200) Subject: SONAR-18872 improve total ncloc computation X-Git-Tag: 10.0.0.68432 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=refs%2Ftags%2F10.0.0.68432;p=sonarqube.git SONAR-18872 improve total ncloc computation --- diff --git a/server/sonar-ce-task-projectanalysis/src/it/java/org/sonar/ce/task/projectanalysis/step/ProjectNclocComputationStepIT.java b/server/sonar-ce-task-projectanalysis/src/it/java/org/sonar/ce/task/projectanalysis/step/ProjectNclocComputationStepIT.java new file mode 100644 index 00000000000..38c925fb7ff --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/it/java/org/sonar/ce/task/projectanalysis/step/ProjectNclocComputationStepIT.java @@ -0,0 +1,65 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.step; + +import org.junit.Rule; +import org.junit.Test; +import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolderRule; +import org.sonar.ce.task.step.TestComputationStepContext; +import org.sonar.db.DbClient; +import org.sonar.db.DbTester; +import org.sonar.db.component.BranchType; +import org.sonar.db.component.ComponentDto; +import org.sonar.db.metric.MetricDto; +import org.sonar.server.project.Project; + +import static java.util.Collections.emptyList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.sonar.api.measures.Metric.ValueType.INT; + +public class ProjectNclocComputationStepIT { + @Rule + public DbTester db = DbTester.create(); + private final DbClient dbClient = db.getDbClient(); + + @Rule + public AnalysisMetadataHolderRule analysisMetadataHolder = new AnalysisMetadataHolderRule(); + + private final ProjectNclocComputationStep step = new ProjectNclocComputationStep(analysisMetadataHolder, dbClient); + + @Test + public void test_computing_branch_ncloc() { + MetricDto ncloc = db.measures().insertMetric(m -> m.setKey("ncloc").setValueType(INT.toString())); + ComponentDto project = db.components().insertPublicProject(); + ComponentDto branch1 = db.components().insertProjectBranch(project, b -> b.setBranchType(BranchType.BRANCH)); + db.measures().insertLiveMeasure(branch1, ncloc, m -> m.setValue(200d)); + ComponentDto branch2 = db.components().insertProjectBranch(project, b -> b.setBranchType(BranchType.BRANCH)); + db.measures().insertLiveMeasure(branch2, ncloc, m -> m.setValue(10d)); + analysisMetadataHolder.setProject(new Project(project.uuid(), project.getKey(), project.name(), project.description(), emptyList())); + step.execute(TestComputationStepContext.TestStatistics::new); + + assertThat(dbClient.projectDao().getNclocSum(db.getSession())).isEqualTo(200L); + } + + @Test + public void description_is_not_missing() { + assertThat(step.getDescription()).isNotBlank(); + } +} \ No newline at end of file diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ProjectNclocComputationStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ProjectNclocComputationStep.java new file mode 100644 index 00000000000..c2cc101df84 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ProjectNclocComputationStep.java @@ -0,0 +1,51 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.step; + +import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolder; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; + +public class ProjectNclocComputationStep implements ComputationStep { + + private final AnalysisMetadataHolder analysisMetadataHolder; + private final DbClient dbClient; + + public ProjectNclocComputationStep(AnalysisMetadataHolder analysisMetadataHolder, DbClient dbClient) { + this.analysisMetadataHolder = analysisMetadataHolder; + this.dbClient = dbClient; + } + + @Override + public void execute(Context context) { + try (DbSession dbSession = dbClient.openSession(false)) { + String projectUuid = analysisMetadataHolder.getProject().getUuid(); + long maxncloc = dbClient.liveMeasureDao().sumNclocOfBiggestBranchForProject(dbSession, projectUuid); + dbClient.projectDao().updateNcloc(dbSession, projectUuid, maxncloc); + dbSession.commit(); + } + } + + @Override + public String getDescription() { + return "Compute total Project ncloc"; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java index 8267f5a216a..e93279055b4 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java @@ -110,6 +110,7 @@ public class ReportComputationSteps extends AbstractComputationSteps { PurgeDatastoresStep.class, IndexAnalysisStep.class, UpdateNeedIssueSyncStep.class, + ProjectNclocComputationStep.class, PersistPushEventsStep.class, // notifications are sent at the end, so that webapp displays up-to-date information diff --git a/server/sonar-db-dao/src/it/java/org/sonar/db/measure/LiveMeasureDaoIT.java b/server/sonar-db-dao/src/it/java/org/sonar/db/measure/LiveMeasureDaoIT.java index f63a7f5ef1f..e519befcad4 100644 --- a/server/sonar-db-dao/src/it/java/org/sonar/db/measure/LiveMeasureDaoIT.java +++ b/server/sonar-db-dao/src/it/java/org/sonar/db/measure/LiveMeasureDaoIT.java @@ -365,12 +365,9 @@ public class LiveMeasureDaoIT { db.measures().insertLiveMeasure(projectWithLinesButNoLoc, lines, m -> m.setValue(365d)); db.measures().insertLiveMeasure(projectWithLinesButNoLoc, ncloc, m -> m.setValue(0d)); - SumNclocDbQuery query = SumNclocDbQuery.builder() - .setOnlyPrivateProjects(false) - .build(); - long result = underTest.sumNclocOfBiggestBranch(db.getSession(), query); - - assertThat(result).isEqualTo(10L + 200L); + assertThat(underTest.sumNclocOfBiggestBranchForProject(db.getSession(), simpleProject.uuid())).isEqualTo(10L); + assertThat(underTest.sumNclocOfBiggestBranchForProject(db.getSession(), projectWithBiggerBranch.uuid())).isEqualTo(200L); + assertThat(underTest.sumNclocOfBiggestBranchForProject(db.getSession(), projectWithLinesButNoLoc.uuid())).isZero(); } @Test @@ -411,40 +408,11 @@ public class LiveMeasureDaoIT { public void countNcloc_empty() { db.measures().insertMetric(m -> m.setKey("ncloc").setValueType(INT.toString())); db.measures().insertMetric(m -> m.setKey("lines").setValueType(INT.toString())); - SumNclocDbQuery query = SumNclocDbQuery.builder() - .setOnlyPrivateProjects(false) - .build(); - long result = underTest.sumNclocOfBiggestBranch(db.getSession(), query); + long result = underTest.sumNclocOfBiggestBranchForProject(db.getSession(), "non-existing-project-uuid"); assertThat(result).isZero(); } - @Test - public void countNcloc_and_exclude_project() { - MetricDto ncloc = db.measures().insertMetric(m -> m.setKey("ncloc").setValueType(INT.toString())); - - ComponentDto simpleProject = db.components().insertPublicProject(); - db.measures().insertLiveMeasure(simpleProject, ncloc, m -> m.setValue(10d)); - - ComponentDto projectWithBiggerBranch = db.components().insertPublicProject(); - ComponentDto bigBranch = db.components().insertProjectBranch(projectWithBiggerBranch, b -> b.setBranchType(BranchType.BRANCH)); - db.measures().insertLiveMeasure(projectWithBiggerBranch, ncloc, m -> m.setValue(100d)); - db.measures().insertLiveMeasure(bigBranch, ncloc, m -> m.setValue(200d)); - - ComponentDto projectToExclude = db.components().insertPublicProject(); - ComponentDto projectToExcludeBranch = db.components().insertProjectBranch(projectToExclude, b -> b.setBranchType(BranchType.BRANCH)); - db.measures().insertLiveMeasure(projectToExclude, ncloc, m -> m.setValue(300d)); - db.measures().insertLiveMeasure(projectToExcludeBranch, ncloc, m -> m.setValue(400d)); - - SumNclocDbQuery query = SumNclocDbQuery.builder() - .setProjectUuidToExclude(projectToExclude.uuid()) - .setOnlyPrivateProjects(false) - .build(); - long result = underTest.sumNclocOfBiggestBranch(db.getSession(), query); - - assertThat(result).isEqualTo(10L + 200L); - } - @Test public void insert_data() { byte[] data = "text_value".getBytes(StandardCharsets.UTF_8); diff --git a/server/sonar-db-dao/src/it/java/org/sonar/db/project/ProjectDaoIT.java b/server/sonar-db-dao/src/it/java/org/sonar/db/project/ProjectDaoIT.java index 7364e759ef6..05958a69359 100644 --- a/server/sonar-db-dao/src/it/java/org/sonar/db/project/ProjectDaoIT.java +++ b/server/sonar-db-dao/src/it/java/org/sonar/db/project/ProjectDaoIT.java @@ -31,6 +31,7 @@ import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.IntStream; import javax.annotation.Nullable; +import org.assertj.core.api.Assertions; import org.assertj.core.groups.Tuple; import org.junit.Rule; import org.junit.Test; @@ -279,6 +280,31 @@ public class ProjectDaoIT { .containsExactlyInAnyOrderElementsOf(extractComponentUuids(projects)); } + @Test + public void update_ncloc_should_update_project() { + ComponentDto project = db.components().insertPublicProject(); + + projectDao.updateNcloc(db.getSession(), project.uuid(), 10L); + + Assertions.assertThat(projectDao.getNclocSum(db.getSession())).isEqualTo(10L); + } + + @Test + public void getNcloc_sum_compute_correctly_sum_of_projects() { + projectDao.updateNcloc(db.getSession(), db.components().insertPublicProject().uuid(), 1L); + projectDao.updateNcloc(db.getSession(), db.components().insertPublicProject().uuid(), 20L); + projectDao.updateNcloc(db.getSession(), db.components().insertPublicProject().uuid(), 100L); + Assertions.assertThat(projectDao.getNclocSum(db.getSession())).isEqualTo(121L); + } + + @Test + public void getNcloc_sum_compute_correctly_sum_of_projects_while_excluding_project() { + projectDao.updateNcloc(db.getSession(), db.components().insertPublicProject().uuid(), 1L); + projectDao.updateNcloc(db.getSession(), db.components().insertPublicProject().uuid(), 20L); + ComponentDto project3 = db.components().insertPublicProject(); + projectDao.updateNcloc(db.getSession(), project3.uuid(), 100L); + Assertions.assertThat(projectDao.getNclocSum(db.getSession(), project3.uuid())).isEqualTo(21L); + } @Test public void selectAllProjectUuids_shouldOnlyReturnProjectWithTRKQualifier() { ComponentDto application = db.components().insertPrivateApplication(); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDao.java index 9f04ff22b5e..9dcfb0e72de 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDao.java @@ -94,14 +94,8 @@ public class LiveMeasureDao implements Dao { mapper(dbSession).selectTreeByQuery(query, baseComponent.uuid(), query.getUuidPath(baseComponent), resultHandler); } - /** - * Example: - * If Main Branch = 0 LOCs (provisioned but never analyzed) and the "largest branch" is 120 LOCs, I'm expecting to consider the value 120. - * If Main Branch = 100 LOCs and the "largest branch" is 120 LOCs, I'm expecting to consider the value 120. - * If Main Branch = 100 LOCs and the "largest branch" is 80 LOCs, I'm expecting to consider the value 100. - */ - public long sumNclocOfBiggestBranch(DbSession dbSession, SumNclocDbQuery dbQuery) { - Long ncloc = mapper(dbSession).sumNclocOfBiggestBranch(NCLOC_KEY, dbQuery.getOnlyPrivateProjects(), dbQuery.getProjectUuidToExclude()); + public long sumNclocOfBiggestBranchForProject(DbSession dbSession, String projectUuid){ + Long ncloc = mapper(dbSession).sumNclocOfBiggestBranchForProject(projectUuid, NCLOC_KEY); return ncloc == null ? 0L : ncloc; } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureMapper.java index e93f7742a24..9b84c0c3cee 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureMapper.java @@ -21,7 +21,7 @@ package org.sonar.db.measure; import java.util.Collection; import java.util.List; -import javax.annotation.Nullable; +import javax.annotation.CheckForNull; import org.apache.ibatis.annotations.Param; import org.apache.ibatis.session.ResultHandler; @@ -57,10 +57,8 @@ public interface LiveMeasureMapper { @Param("baseUuidPath") String baseUuidPath, ResultHandler resultHandler); - Long sumNclocOfBiggestBranch( - @Param("ncloc") String nclocKey, - @Param("private") Boolean privateProject, - @Nullable @Param("projectUuidToExclude") String projectUuidToExclude); + @CheckForNull + Long sumNclocOfBiggestBranchForProject(@Param("projectUuid") String projectUuid, @Param("ncloc") String nclocKey); List getLargestBranchNclocPerProject(@Param("nclocUuid") String nclocUuid); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectDao.java index 30f3796acb8..11f5c17c37f 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectDao.java @@ -23,6 +23,7 @@ import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.Set; +import javax.annotation.Nullable; import org.sonar.api.utils.System2; import org.sonar.db.Dao; import org.sonar.db.DbSession; @@ -126,4 +127,16 @@ public class ProjectDao implements Dao { Set languageFilters = Set.of(language + "=%", "%;" + language + "=%"); return mapper(session).selectProjectUuidsAssociatedToDefaultQualityProfileByLanguage(languageFilters); } + + public void updateNcloc(DbSession dbSession, String projectUuid, long ncloc) { + mapper(dbSession).updateNcloc(projectUuid, ncloc); + } + + public long getNclocSum(DbSession dbSession) { + return getNclocSum(dbSession, null); + } + + public long getNclocSum(DbSession dbSession, @Nullable String projectUuidToExclude) { + return Optional.ofNullable(mapper(dbSession).getNclocSum(projectUuidToExclude)).orElse(0L); + } } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectMapper.java index 58e635cd016..3253c1578b3 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/project/ProjectMapper.java @@ -23,6 +23,7 @@ import java.util.Collection; import java.util.List; import java.util.Set; import javax.annotation.CheckForNull; +import javax.annotation.Nullable; import org.apache.ibatis.annotations.Param; public interface ProjectMapper { @@ -62,4 +63,10 @@ public interface ProjectMapper { List selectAllProjectUuids(); Set selectProjectUuidsAssociatedToDefaultQualityProfileByLanguage(@Param("languageFilters") Set languageFilters); + + void updateNcloc(@Param("projectUuid") String projectUuid, @Param("ncloc") long ncloc); + + @CheckForNull + Long getNclocSum(@Nullable @Param("projectUuidToExclude") String projectUuidToExclude); + } diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/LiveMeasureMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/LiveMeasureMapper.xml index c4d49599fc1..1395671580d 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/LiveMeasureMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/measure/LiveMeasureMapper.xml @@ -61,8 +61,8 @@ and lm.component_uuid = #{componentUuid, jdbcType=VARCHAR} - + select sumncloc.maxncloc from ( select b.project_uuid as projectUuid, max(lm.value) as maxncloc from live_measures lm inner join metrics m on m.uuid = lm.metric_uuid @@ -70,12 +70,7 @@ inner join projects p on p.uuid = b.project_uuid and p.qualifier = 'TRK' m.name = #{ncloc, jdbcType=VARCHAR} - - and p.private=${_true} - - - and b.project_uuid <> #{projectUuidToExclude,jdbcType=VARCHAR} - + and b.project_uuid = #{projectUuid,jdbcType=VARCHAR} group by b.project_uuid ) sumncloc diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/project/ProjectMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/project/ProjectMapper.xml index 9f32c5a8520..baae55a33f2 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/project/ProjectMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/project/ProjectMapper.xml @@ -180,4 +180,19 @@ uuid = #{uuid,jdbcType=VARCHAR} + + + update projects set + ncloc = #{ncloc,jdbcType=BIGINT} + where + uuid = #{projectUuid,jdbcType=VARCHAR} + + + + diff --git a/server/sonar-db-dao/src/schema/schema-sq.ddl b/server/sonar-db-dao/src/schema/schema-sq.ddl index 0662a2a6a89..f3b228c4440 100644 --- a/server/sonar-db-dao/src/schema/schema-sq.ddl +++ b/server/sonar-db-dao/src/schema/schema-sq.ddl @@ -716,7 +716,8 @@ CREATE TABLE "PROJECTS"( "PRIVATE" BOOLEAN NOT NULL, "TAGS" CHARACTER VARYING(500), "CREATED_AT" BIGINT, - "UPDATED_AT" BIGINT NOT NULL + "UPDATED_AT" BIGINT NOT NULL, + "NCLOC" BIGINT ); ALTER TABLE "PROJECTS" ADD CONSTRAINT "PK_NEW_PROJECTS" PRIMARY KEY("UUID"); CREATE UNIQUE INDEX "UNIQ_PROJECTS_KEE" ON "PROJECTS"("KEE" NULLS FIRST); diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v100/AddNclocToProjects.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v100/AddNclocToProjects.java new file mode 100644 index 00000000000..c39dc36e035 --- /dev/null +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v100/AddNclocToProjects.java @@ -0,0 +1,56 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.platform.db.migration.version.v100; + +import java.sql.SQLException; +import org.sonar.db.Database; +import org.sonar.db.DatabaseUtils; +import org.sonar.server.platform.db.migration.def.BigIntegerColumnDef; +import org.sonar.server.platform.db.migration.sql.AddColumnsBuilder; +import org.sonar.server.platform.db.migration.step.DdlChange; + +public class AddNclocToProjects extends DdlChange { + + public static final String PROJECT_TABLE_NAME = "projects"; + public static final String NCLOC_COLUMN_NAME = "ncloc"; + + public AddNclocToProjects(Database db) { + super(db); + } + + @Override + public void execute(Context context) throws SQLException { + if (checkIfColumnExists()) { + return; + } + BigIntegerColumnDef columnDef = BigIntegerColumnDef.newBigIntegerColumnDefBuilder().setColumnName(NCLOC_COLUMN_NAME).setIsNullable(true).build(); + String request = new AddColumnsBuilder(getDialect(), PROJECT_TABLE_NAME).addColumn(columnDef).build(); + context.execute(request); + } + + public boolean checkIfColumnExists() throws SQLException { + try (var connection = getDatabase().getDataSource().getConnection()) { + if (DatabaseUtils.tableColumnExists(connection, PROJECT_TABLE_NAME, NCLOC_COLUMN_NAME)) { + return true; + } + } + return false; + } +} diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v100/DbVersion100.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v100/DbVersion100.java index 1217495bcc2..ba6d6a3169f 100644 --- a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v100/DbVersion100.java +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v100/DbVersion100.java @@ -56,6 +56,8 @@ public class DbVersion100 implements DbVersion { .add(10_0_012, "Log a warning message if 'sonar.scim.enabled' is used", LogMessageIfSonarScimEnabledPresentProperty.class) .add(10_0_013, "Drop 'sonar.scim.enabled' property", DropSonarScimEnabledProperty.class) .add(10_0_014, "Drop any SCIM User provisioning, turning all users local", DropScimUserProvisioning.class) + .add(10_0_015, "Add ncloc to 'Projects' table", AddNclocToProjects.class) + .add(10_0_016, "Populate ncloc in 'Projects' table", PopulateNclocForForProjects.class) ; } } diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v100/PopulateNclocForForProjects.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v100/PopulateNclocForForProjects.java new file mode 100644 index 00000000000..ffb3fb0e58d --- /dev/null +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v100/PopulateNclocForForProjects.java @@ -0,0 +1,56 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.platform.db.migration.version.v100; + +import java.sql.SQLException; +import org.sonar.db.Database; +import org.sonar.server.platform.db.migration.step.DataChange; +import org.sonar.server.platform.db.migration.step.MassUpdate; + +public class PopulateNclocForForProjects extends DataChange { + + private static final String SELECT_QUERY = """ + SELECT b.project_uuid AS projectUuid, max(lm.value) AS maxncloc + FROM live_measures lm + INNER JOIN metrics m ON m.uuid = lm.metric_uuid + INNER JOIN project_branches b ON b.uuid = lm.component_uuid + INNER JOIN projects p on p.uuid = b.project_uuid and p.qualifier = 'TRK' + WHERE m.name = 'ncloc' + GROUP BY b.project_uuid + """; + + public PopulateNclocForForProjects(Database db) { + super(db); + } + + @Override + protected void execute(Context context) throws SQLException { + MassUpdate massUpdate = context.prepareMassUpdate(); + massUpdate.select(SELECT_QUERY); + massUpdate.update("update projects set ncloc = ? where uuid = ?"); + massUpdate.execute((row, update) -> { + String uuid = row.getString(1); + Long ncloc = row.getLong(2); + update.setLong(1, ncloc); + update.setString(2, uuid); + return true; + }); + } +} diff --git a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v100/AddNclocToProjectsTest.java b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v100/AddNclocToProjectsTest.java new file mode 100644 index 00000000000..c67930e4be6 --- /dev/null +++ b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v100/AddNclocToProjectsTest.java @@ -0,0 +1,54 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.platform.db.migration.version.v100; + +import java.sql.SQLException; +import java.sql.Types; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.db.CoreDbTester; +import org.sonar.server.platform.db.migration.step.DdlChange; + +public class AddNclocToProjectsTest { + + private static final String TABLE_NAME = "projects"; + private static final String COLUMN_NAME = "ncloc"; + + @Rule + public final CoreDbTester db = CoreDbTester.createForSchema(AddNclocToProjectsTest.class, "schema.sql"); + private final DdlChange underTest = new AddNclocToProjects(db.database()); + + @Test + public void add_column() throws SQLException { + db.assertColumnDoesNotExist(TABLE_NAME, COLUMN_NAME); + db.assertColumnDoesNotExist(TABLE_NAME, COLUMN_NAME); + underTest.execute(); + db.assertColumnDefinition(TABLE_NAME, COLUMN_NAME, Types.BIGINT, null, true); + } + + @Test + public void migration_is_reentrant() throws SQLException { + db.assertColumnDoesNotExist(TABLE_NAME, COLUMN_NAME); + underTest.execute(); + underTest.execute(); + db.assertColumnDefinition(TABLE_NAME, COLUMN_NAME, Types.BIGINT, null, true); + } + +} \ No newline at end of file diff --git a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v100/PopulateNclocForForProjectsTest.java b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v100/PopulateNclocForForProjectsTest.java new file mode 100644 index 00000000000..66527f7d884 --- /dev/null +++ b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v100/PopulateNclocForForProjectsTest.java @@ -0,0 +1,144 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.platform.db.migration.version.v100; + +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.core.util.UuidFactory; +import org.sonar.core.util.UuidFactoryFast; +import org.sonar.db.CoreDbTester; +import org.sonar.server.platform.db.migration.step.DataChange; + +import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; +import static org.assertj.core.api.Assertions.assertThat; + +public class PopulateNclocForForProjectsTest { + + private final UuidFactory uuidFactory = UuidFactoryFast.getInstance(); + + @Rule + public CoreDbTester db = CoreDbTester.createForSchema(PopulateNclocForForProjectsTest.class, "schema.sql"); + + private final DataChange underTest = new PopulateNclocForForProjects(db.database()); + + @Test + public void migration_populates_ncloc_for_projects() throws SQLException { + Map expectedNclocByProjectUuid = populateData(); + underTest.execute(); + verifyNclocCorrectlyPopulatedForProjects(expectedNclocByProjectUuid); + } + + @Test + public void migration_should_be_reentrant() throws SQLException { + Map expectedNclocByProjectUuid = populateData(); + underTest.execute(); + // re-entrant + underTest.execute(); + verifyNclocCorrectlyPopulatedForProjects(expectedNclocByProjectUuid); + } + + private Map populateData() { + String nclocMetricUuid = insertMetric("ncloc"); + + String projectUuid1 = insertProject(); + String project1Branch1 = insertProjectBranch(projectUuid1); + String project1Branch2 = insertProjectBranch(projectUuid1); + + long project1maxNcloc = 100; + insertLiveMeasure(nclocMetricUuid, projectUuid1, project1Branch1, 80L); + insertLiveMeasure(nclocMetricUuid, projectUuid1, project1Branch2, project1maxNcloc); + + String otherMetricUuid = insertMetric("other"); + insertLiveMeasure(otherMetricUuid, projectUuid1, project1Branch1, 5000L); + insertLiveMeasure(otherMetricUuid, projectUuid1, project1Branch2, 6000L); + + String projectUuid2 = insertProject(); + String project2Branch1 = insertProjectBranch(projectUuid2); + String project2Branch2 = insertProjectBranch(projectUuid2); + String project2Branch3 = insertProjectBranch(projectUuid2); + + long project2maxNcloc = 60; + insertLiveMeasure(nclocMetricUuid, projectUuid2, project2Branch1, 20L); + insertLiveMeasure(nclocMetricUuid, projectUuid2, project2Branch2, 50L); + insertLiveMeasure(nclocMetricUuid, projectUuid2, project2Branch3, project2maxNcloc); + + return Map.of(projectUuid1, project1maxNcloc, projectUuid2, project2maxNcloc); + } + + private void verifyNclocCorrectlyPopulatedForProjects(Map expectedNclocByProjectUuid) { + for (Map.Entry entry : expectedNclocByProjectUuid.entrySet()) { + String query = String.format("select ncloc from projects where uuid='%s'", entry.getKey()); + Long nclocFromProject = (Long) db.selectFirst(query).get("NCLOC"); + assertThat(nclocFromProject).isEqualTo(entry.getValue()); + } + } + + private String insertMetric(String name) { + Map map = new HashMap<>(); + String uuid = uuidFactory.create(); + map.put("UUID", uuid); + map.put("NAME", name); + db.executeInsert("metrics", map); + return uuid; + } + + private String insertProject() { + Map map = new HashMap<>(); + String uuid = uuidFactory.create(); + map.put("UUID", uuid); + map.put("KEE", randomAlphabetic(20)); + map.put("QUALIFIER", "TRK"); + map.put("PRIVATE", true); + map.put("UPDATED_AT", System.currentTimeMillis()); + db.executeInsert("projects", map); + return uuid; + } + + private String insertProjectBranch(String projectUuid) { + Map map = new HashMap<>(); + String uuid = uuidFactory.create(); + map.put("UUID", uuid); + map.put("PROJECT_UUID", projectUuid); + map.put("KEE", randomAlphabetic(20)); + map.put("BRANCH_TYPE", "PULL_REQUEST"); + map.put("UPDATED_AT", System.currentTimeMillis()); + map.put("CREATED_AT", System.currentTimeMillis()); + map.put("NEED_ISSUE_SYNC", false); + db.executeInsert("project_branches", map); + return uuid; + } + + private void insertLiveMeasure(String metricUuid, String projectUuid, String componentUuid, Long value) { + Map map = new HashMap<>(); + String uuid = uuidFactory.create(); + map.put("UUID", uuid); + map.put("PROJECT_UUID", projectUuid); + map.put("COMPONENT_UUID", componentUuid); + map.put("METRIC_UUID", metricUuid); + map.put("VALUE", value); + map.put("UPDATED_AT", System.currentTimeMillis()); + map.put("CREATED_AT", System.currentTimeMillis()); + db.executeInsert("live_measures", map); + } + +} diff --git a/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v100/AddNclocToProjectsTest/schema.sql b/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v100/AddNclocToProjectsTest/schema.sql new file mode 100644 index 00000000000..5cc62fcd124 --- /dev/null +++ b/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v100/AddNclocToProjectsTest/schema.sql @@ -0,0 +1,15 @@ + +CREATE TABLE "PROJECTS"( + "UUID" CHARACTER VARYING(40) NOT NULL, + "KEE" CHARACTER VARYING(400) NOT NULL, + "QUALIFIER" CHARACTER VARYING(10) NOT NULL, + "NAME" CHARACTER VARYING(2000), + "DESCRIPTION" CHARACTER VARYING(2000), + "PRIVATE" BOOLEAN NOT NULL, + "TAGS" CHARACTER VARYING(500), + "CREATED_AT" BIGINT, + "UPDATED_AT" BIGINT NOT NULL +); +ALTER TABLE "PROJECTS" ADD CONSTRAINT "PK_NEW_PROJECTS" PRIMARY KEY("UUID"); +CREATE UNIQUE INDEX "UNIQ_PROJECTS_KEE" ON "PROJECTS"("KEE" NULLS FIRST); +CREATE INDEX "IDX_QUALIFIER" ON "PROJECTS"("QUALIFIER" NULLS FIRST); diff --git a/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v100/PopulateNclocForForProjectsTest/schema.sql b/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v100/PopulateNclocForForProjectsTest/schema.sql new file mode 100644 index 00000000000..f69434e8266 --- /dev/null +++ b/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v100/PopulateNclocForForProjectsTest/schema.sql @@ -0,0 +1,67 @@ +CREATE TABLE "PROJECTS"( + "UUID" CHARACTER VARYING(40) NOT NULL, + "KEE" CHARACTER VARYING(400) NOT NULL, + "QUALIFIER" CHARACTER VARYING(10) NOT NULL, + "NAME" CHARACTER VARYING(2000), + "DESCRIPTION" CHARACTER VARYING(2000), + "PRIVATE" BOOLEAN NOT NULL, + "TAGS" CHARACTER VARYING(500), + "CREATED_AT" BIGINT, + "UPDATED_AT" BIGINT NOT NULL, + "NCLOC" BIGINT +); +ALTER TABLE "PROJECTS" ADD CONSTRAINT "PK_NEW_PROJECTS" PRIMARY KEY("UUID"); +CREATE UNIQUE INDEX "UNIQ_PROJECTS_KEE" ON "PROJECTS"("KEE" NULLS FIRST); +CREATE INDEX "IDX_QUALIFIER" ON "PROJECTS"("QUALIFIER" NULLS FIRST); + +CREATE TABLE "PROJECT_BRANCHES"( + "UUID" CHARACTER VARYING(50) NOT NULL, + "PROJECT_UUID" CHARACTER VARYING(50) NOT NULL, + "KEE" CHARACTER VARYING(255) NOT NULL, + "BRANCH_TYPE" CHARACTER VARYING(12) NOT NULL, + "MERGE_BRANCH_UUID" CHARACTER VARYING(50), + "PULL_REQUEST_BINARY" BINARY LARGE OBJECT, + "MANUAL_BASELINE_ANALYSIS_UUID" CHARACTER VARYING(40), + "CREATED_AT" BIGINT NOT NULL, + "UPDATED_AT" BIGINT NOT NULL, + "EXCLUDE_FROM_PURGE" BOOLEAN DEFAULT FALSE NOT NULL, + "NEED_ISSUE_SYNC" BOOLEAN NOT NULL +); +ALTER TABLE "PROJECT_BRANCHES" ADD CONSTRAINT "PK_PROJECT_BRANCHES" PRIMARY KEY("UUID"); +CREATE UNIQUE INDEX "UNIQ_PROJECT_BRANCHES" ON "PROJECT_BRANCHES"("BRANCH_TYPE" NULLS FIRST, "PROJECT_UUID" NULLS FIRST, "KEE" NULLS FIRST); + +CREATE TABLE "LIVE_MEASURES"( + "UUID" CHARACTER VARYING(40) NOT NULL, + "PROJECT_UUID" CHARACTER VARYING(50) NOT NULL, + "COMPONENT_UUID" CHARACTER VARYING(50) NOT NULL, + "METRIC_UUID" CHARACTER VARYING(40) NOT NULL, + "VALUE" DOUBLE PRECISION, + "TEXT_VALUE" CHARACTER VARYING(4000), + "MEASURE_DATA" BINARY LARGE OBJECT, + "UPDATE_MARKER" CHARACTER VARYING(40), + "CREATED_AT" BIGINT NOT NULL, + "UPDATED_AT" BIGINT NOT NULL +); +ALTER TABLE "LIVE_MEASURES" ADD CONSTRAINT "PK_LIVE_MEASURES" PRIMARY KEY("UUID"); +CREATE INDEX "LIVE_MEASURES_PROJECT" ON "LIVE_MEASURES"("PROJECT_UUID" NULLS FIRST); +CREATE UNIQUE INDEX "LIVE_MEASURES_COMPONENT" ON "LIVE_MEASURES"("COMPONENT_UUID" NULLS FIRST, "METRIC_UUID" NULLS FIRST); + +CREATE TABLE "METRICS"( + "UUID" CHARACTER VARYING(40) NOT NULL, + "NAME" CHARACTER VARYING(64) NOT NULL, + "DESCRIPTION" CHARACTER VARYING(255), + "DIRECTION" INTEGER DEFAULT 0 NOT NULL, + "DOMAIN" CHARACTER VARYING(64), + "SHORT_NAME" CHARACTER VARYING(64), + "QUALITATIVE" BOOLEAN DEFAULT FALSE NOT NULL, + "VAL_TYPE" CHARACTER VARYING(8), + "ENABLED" BOOLEAN DEFAULT TRUE, + "WORST_VALUE" DOUBLE PRECISION, + "BEST_VALUE" DOUBLE PRECISION, + "OPTIMIZED_BEST_VALUE" BOOLEAN, + "HIDDEN" BOOLEAN, + "DELETE_HISTORICAL_DATA" BOOLEAN, + "DECIMAL_SCALE" INTEGER +); +ALTER TABLE "METRICS" ADD CONSTRAINT "PK_METRICS" PRIMARY KEY("UUID"); +CREATE UNIQUE INDEX "METRICS_UNIQUE_NAME" ON "METRICS"("NAME" NULLS FIRST); diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/StatisticsSupport.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/StatisticsSupport.java index 29325634dce..4d8f07d5c06 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/StatisticsSupport.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/StatisticsSupport.java @@ -21,7 +21,6 @@ package org.sonar.server.platform; import org.sonar.db.DbClient; import org.sonar.db.DbSession; -import org.sonar.db.measure.SumNclocDbQuery; public class StatisticsSupport { @@ -33,10 +32,7 @@ public class StatisticsSupport { public long getLinesOfCode(){ try (DbSession dbSession = dbClient.openSession(false)) { - SumNclocDbQuery query = SumNclocDbQuery.builder() - .setOnlyPrivateProjects(false) - .build(); - return dbClient.liveMeasureDao().sumNclocOfBiggestBranch(dbSession, query); + return dbClient.projectDao().getNclocSum(dbSession); } } } diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/StatisticsSupportTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/StatisticsSupportTest.java index 872862a0699..052ab11bbea 100644 --- a/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/StatisticsSupportTest.java +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/platform/StatisticsSupportTest.java @@ -22,7 +22,6 @@ package org.sonar.server.platform; import org.junit.Test; import org.sonar.db.DbClient; import org.sonar.db.DbSession; -import org.sonar.db.measure.SumNclocDbQuery; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; @@ -37,7 +36,7 @@ public class StatisticsSupportTest { @Test public void should_return_metric_from_liveMeasureDao() { - when(dbClient.liveMeasureDao().sumNclocOfBiggestBranch(any(DbSession.class), any(SumNclocDbQuery.class))).thenReturn(1800999L); + when(dbClient.projectDao().getNclocSum(any(DbSession.class))).thenReturn(1800999L); long linesOfCode = statisticsSupport.getLinesOfCode(); diff --git a/server/sonar-webserver-webapi/src/it/java/org/sonar/server/ui/ws/MarketplaceActionIT.java b/server/sonar-webserver-webapi/src/it/java/org/sonar/server/ui/ws/MarketplaceActionIT.java index 2b6f01fdd68..001e1d35889 100644 --- a/server/sonar-webserver-webapi/src/it/java/org/sonar/server/ui/ws/MarketplaceActionIT.java +++ b/server/sonar-webserver-webapi/src/it/java/org/sonar/server/ui/ws/MarketplaceActionIT.java @@ -28,7 +28,6 @@ import org.sonar.api.server.ws.WebService; import org.sonar.db.DbClient; import org.sonar.db.DbTester; import org.sonar.db.component.ComponentDto; -import org.sonar.db.metric.MetricDto; import org.sonar.server.exceptions.ForbiddenException; import org.sonar.server.exceptions.UnauthorizedException; import org.sonar.server.tester.UserSessionRule; @@ -40,8 +39,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY; -import static org.sonar.api.measures.Metric.ValueType.INT; import static org.sonar.test.JsonAssert.assertJson; @RunWith(DataProviderRunner.class) @@ -120,7 +117,7 @@ public class MarketplaceActionIT { private void setNcloc(double ncloc) { ComponentDto project = db.components().insertPublicProject(); - MetricDto nclocMetric = db.measures().insertMetric(m -> m.setValueType(INT.toString()).setKey(NCLOC_KEY)); - db.measures().insertLiveMeasure(project, nclocMetric, m -> m.setValue(ncloc)); + db.getDbClient().projectDao().updateNcloc(db.getSession(), project.uuid(), (long) ncloc); + db.commit(); } } diff --git a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/ui/ws/MarketplaceAction.java b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/ui/ws/MarketplaceAction.java index 7c4f7b3b8bb..918f8a49b47 100644 --- a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/ui/ws/MarketplaceAction.java +++ b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/ui/ws/MarketplaceAction.java @@ -25,7 +25,6 @@ import org.sonar.api.server.ws.Response; import org.sonar.api.server.ws.WebService; import org.sonar.db.DbClient; import org.sonar.db.DbSession; -import org.sonar.db.measure.SumNclocDbQuery; import org.sonar.server.user.UserSession; import org.sonarqube.ws.Navigation; @@ -68,10 +67,7 @@ public class MarketplaceAction implements NavigationWsAction { private long computeNcloc() { try (DbSession dbSession = dbClient.openSession(false)) { - SumNclocDbQuery query = SumNclocDbQuery.builder() - .setOnlyPrivateProjects(false) - .build(); - return dbClient.liveMeasureDao().sumNclocOfBiggestBranch(dbSession, query); + return dbClient.projectDao().getNclocSum(dbSession); } } }