SONAR-18872 improve total ncloc computation10.0.0.68432

7 files changed, 394 insertions, 0 deletions

diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v100/AddNclocToProjects.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v100/AddNclocToProjects.java
new file mode 100644
index 00000000000..c39dc36e035
--- /dev/null
+++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v100/AddNclocToProjects.java
@@ -0,0 +1,56 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.platform.db.migration.version.v100;
+
+import java.sql.SQLException;
+import org.sonar.db.Database;
+import org.sonar.db.DatabaseUtils;
+import org.sonar.server.platform.db.migration.def.BigIntegerColumnDef;
+import org.sonar.server.platform.db.migration.sql.AddColumnsBuilder;
+import org.sonar.server.platform.db.migration.step.DdlChange;
+
+public class AddNclocToProjects extends DdlChange {
+
+  public static final String PROJECT_TABLE_NAME = "projects";
+  public static final String NCLOC_COLUMN_NAME = "ncloc";
+
+  public AddNclocToProjects(Database db) {
+    super(db);
+  }
+
+  @Override
+  public void execute(Context context) throws SQLException {
+    if (checkIfColumnExists()) {
+      return;
+    }
+    BigIntegerColumnDef columnDef = BigIntegerColumnDef.newBigIntegerColumnDefBuilder().setColumnName(NCLOC_COLUMN_NAME).setIsNullable(true).build();
+    String request = new AddColumnsBuilder(getDialect(), PROJECT_TABLE_NAME).addColumn(columnDef).build();
+    context.execute(request);
+  }
+
+  public boolean checkIfColumnExists() throws SQLException {
+    try (var connection = getDatabase().getDataSource().getConnection()) {
+      if (DatabaseUtils.tableColumnExists(connection, PROJECT_TABLE_NAME, NCLOC_COLUMN_NAME)) {
+        return true;
+      }
+    }
+    return false;
+  }
+}
diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v100/DbVersion100.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v100/DbVersion100.java
index 1217495bcc2..ba6d6a3169f 100644
--- a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v100/DbVersion100.java
+++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v100/DbVersion100.java
@@ -56,6 +56,8 @@ public class DbVersion100 implements DbVersion {
       .add(10_0_012, "Log a warning message if 'sonar.scim.enabled' is used", LogMessageIfSonarScimEnabledPresentProperty.class)
       .add(10_0_013, "Drop 'sonar.scim.enabled' property", DropSonarScimEnabledProperty.class)
       .add(10_0_014, "Drop any SCIM User provisioning, turning all users local", DropScimUserProvisioning.class)
+      .add(10_0_015, "Add ncloc to 'Projects' table", AddNclocToProjects.class)
+      .add(10_0_016, "Populate ncloc in 'Projects' table", PopulateNclocForForProjects.class)
     ;
   }
 }
diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v100/PopulateNclocForForProjects.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v100/PopulateNclocForForProjects.java
new file mode 100644
index 00000000000..ffb3fb0e58d
--- /dev/null
+++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v100/PopulateNclocForForProjects.java
@@ -0,0 +1,56 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.platform.db.migration.version.v100;
+
+import java.sql.SQLException;
+import org.sonar.db.Database;
+import org.sonar.server.platform.db.migration.step.DataChange;
+import org.sonar.server.platform.db.migration.step.MassUpdate;
+
+public class PopulateNclocForForProjects extends DataChange {
+
+  private static final String SELECT_QUERY = """
+    SELECT b.project_uuid AS projectUuid, max(lm.value) AS maxncloc
+    FROM live_measures lm
+    INNER JOIN metrics m ON m.uuid = lm.metric_uuid
+    INNER JOIN project_branches b ON b.uuid = lm.component_uuid
+    INNER JOIN projects p on p.uuid = b.project_uuid and p.qualifier = 'TRK'
+    WHERE m.name  = 'ncloc'
+    GROUP BY b.project_uuid
+    """;
+
+  public PopulateNclocForForProjects(Database db) {
+    super(db);
+  }
+
+  @Override
+  protected void execute(Context context) throws SQLException {
+    MassUpdate massUpdate = context.prepareMassUpdate();
+    massUpdate.select(SELECT_QUERY);
+    massUpdate.update("update projects set ncloc = ? where uuid = ?");
+    massUpdate.execute((row, update) -> {
+      String uuid = row.getString(1);
+      Long ncloc = row.getLong(2);
+      update.setLong(1, ncloc);
+      update.setString(2, uuid);
+      return true;
+    });
+  }
+}
diff --git a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v100/AddNclocToProjectsTest.java b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v100/AddNclocToProjectsTest.java
new file mode 100644
index 00000000000..c67930e4be6
--- /dev/null
+++ b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v100/AddNclocToProjectsTest.java
@@ -0,0 +1,54 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.platform.db.migration.version.v100;
+
+import java.sql.SQLException;
+import java.sql.Types;
+import org.junit.Rule;
+import org.junit.Test;
+import org.sonar.db.CoreDbTester;
+import org.sonar.server.platform.db.migration.step.DdlChange;
+
+public class AddNclocToProjectsTest {
+
+  private static final String TABLE_NAME = "projects";
+  private static final String COLUMN_NAME = "ncloc";
+
+  @Rule
+  public final CoreDbTester db = CoreDbTester.createForSchema(AddNclocToProjectsTest.class, "schema.sql");
+  private final DdlChange underTest = new AddNclocToProjects(db.database());
+
+  @Test
+  public void add_column() throws SQLException {
+    db.assertColumnDoesNotExist(TABLE_NAME, COLUMN_NAME);
+    db.assertColumnDoesNotExist(TABLE_NAME, COLUMN_NAME);
+    underTest.execute();
+    db.assertColumnDefinition(TABLE_NAME, COLUMN_NAME, Types.BIGINT, null, true);
+  }
+
+  @Test
+  public void migration_is_reentrant() throws SQLException {
+    db.assertColumnDoesNotExist(TABLE_NAME, COLUMN_NAME);
+    underTest.execute();
+    underTest.execute();
+    db.assertColumnDefinition(TABLE_NAME, COLUMN_NAME, Types.BIGINT, null, true);
+  }
+
+}
\ No newline at end of file
diff --git a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v100/PopulateNclocForForProjectsTest.java b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v100/PopulateNclocForForProjectsTest.java
new file mode 100644
index 00000000000..66527f7d884
--- /dev/null
+++ b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v100/PopulateNclocForForProjectsTest.java
@@ -0,0 +1,144 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.platform.db.migration.version.v100;
+
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
+import org.junit.Rule;
+import org.junit.Test;
+import org.sonar.core.util.UuidFactory;
+import org.sonar.core.util.UuidFactoryFast;
+import org.sonar.db.CoreDbTester;
+import org.sonar.server.platform.db.migration.step.DataChange;
+
+import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class PopulateNclocForForProjectsTest {
+
+  private final UuidFactory uuidFactory = UuidFactoryFast.getInstance();
+
+  @Rule
+  public CoreDbTester db = CoreDbTester.createForSchema(PopulateNclocForForProjectsTest.class, "schema.sql");
+
+  private final DataChange underTest = new PopulateNclocForForProjects(db.database());
+
+  @Test
+  public void migration_populates_ncloc_for_projects() throws SQLException {
+    Map<String, Long> expectedNclocByProjectUuid = populateData();
+    underTest.execute();
+    verifyNclocCorrectlyPopulatedForProjects(expectedNclocByProjectUuid);
+  }
+
+  @Test
+  public void migration_should_be_reentrant() throws SQLException {
+    Map<String, Long> expectedNclocByProjectUuid = populateData();
+    underTest.execute();
+    // re-entrant
+    underTest.execute();
+    verifyNclocCorrectlyPopulatedForProjects(expectedNclocByProjectUuid);
+  }
+
+  private Map<String, Long> populateData() {
+    String nclocMetricUuid = insertMetric("ncloc");
+
+    String projectUuid1 = insertProject();
+    String project1Branch1 = insertProjectBranch(projectUuid1);
+    String project1Branch2 = insertProjectBranch(projectUuid1);
+
+    long project1maxNcloc = 100;
+    insertLiveMeasure(nclocMetricUuid, projectUuid1, project1Branch1, 80L);
+    insertLiveMeasure(nclocMetricUuid, projectUuid1, project1Branch2, project1maxNcloc);
+
+    String otherMetricUuid = insertMetric("other");
+    insertLiveMeasure(otherMetricUuid, projectUuid1, project1Branch1, 5000L);
+    insertLiveMeasure(otherMetricUuid, projectUuid1, project1Branch2, 6000L);
+
+    String projectUuid2 = insertProject();
+    String project2Branch1 = insertProjectBranch(projectUuid2);
+    String project2Branch2 = insertProjectBranch(projectUuid2);
+    String project2Branch3 = insertProjectBranch(projectUuid2);
+
+    long project2maxNcloc = 60;
+    insertLiveMeasure(nclocMetricUuid, projectUuid2, project2Branch1, 20L);
+    insertLiveMeasure(nclocMetricUuid, projectUuid2, project2Branch2, 50L);
+    insertLiveMeasure(nclocMetricUuid, projectUuid2, project2Branch3, project2maxNcloc);
+
+    return Map.of(projectUuid1, project1maxNcloc, projectUuid2, project2maxNcloc);
+  }
+
+  private void verifyNclocCorrectlyPopulatedForProjects(Map<String, Long> expectedNclocByProjectUuid) {
+    for (Map.Entry<String, Long> entry : expectedNclocByProjectUuid.entrySet()) {
+      String query = String.format("select ncloc from projects where uuid='%s'", entry.getKey());
+      Long nclocFromProject = (Long) db.selectFirst(query).get("NCLOC");
+      assertThat(nclocFromProject).isEqualTo(entry.getValue());
+    }
+  }
+
+  private String insertMetric(String name) {
+    Map<String, Object> map = new HashMap<>();
+    String uuid = uuidFactory.create();
+    map.put("UUID", uuid);
+    map.put("NAME", name);
+    db.executeInsert("metrics", map);
+    return uuid;
+  }
+
+  private String insertProject() {
+    Map<String, Object> map = new HashMap<>();
+    String uuid = uuidFactory.create();
+    map.put("UUID", uuid);
+    map.put("KEE", randomAlphabetic(20));
+    map.put("QUALIFIER", "TRK");
+    map.put("PRIVATE", true);
+    map.put("UPDATED_AT", System.currentTimeMillis());
+    db.executeInsert("projects", map);
+    return uuid;
+  }
+
+  private String insertProjectBranch(String projectUuid) {
+    Map<String, Object> map = new HashMap<>();
+    String uuid = uuidFactory.create();
+    map.put("UUID", uuid);
+    map.put("PROJECT_UUID", projectUuid);
+    map.put("KEE", randomAlphabetic(20));
+    map.put("BRANCH_TYPE", "PULL_REQUEST");
+    map.put("UPDATED_AT", System.currentTimeMillis());
+    map.put("CREATED_AT", System.currentTimeMillis());
+    map.put("NEED_ISSUE_SYNC", false);
+    db.executeInsert("project_branches", map);
+    return uuid;
+  }
+
+  private void insertLiveMeasure(String metricUuid, String projectUuid, String componentUuid, Long value) {
+    Map<String, Object> map = new HashMap<>();
+    String uuid = uuidFactory.create();
+    map.put("UUID", uuid);
+    map.put("PROJECT_UUID", projectUuid);
+    map.put("COMPONENT_UUID", componentUuid);
+    map.put("METRIC_UUID", metricUuid);
+    map.put("VALUE", value);
+    map.put("UPDATED_AT", System.currentTimeMillis());
+    map.put("CREATED_AT", System.currentTimeMillis());
+    db.executeInsert("live_measures", map);
+  }
+
+}
diff --git a/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v100/AddNclocToProjectsTest/schema.sql b/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v100/AddNclocToProjectsTest/schema.sql
new file mode 100644
index 00000000000..5cc62fcd124
--- /dev/null
+++ b/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v100/AddNclocToProjectsTest/schema.sql
@@ -0,0 +1,15 @@
+
+CREATE TABLE "PROJECTS"(
+    "UUID" CHARACTER VARYING(40) NOT NULL,
+    "KEE" CHARACTER VARYING(400) NOT NULL,
+    "QUALIFIER" CHARACTER VARYING(10) NOT NULL,
+    "NAME" CHARACTER VARYING(2000),
+    "DESCRIPTION" CHARACTER VARYING(2000),
+    "PRIVATE" BOOLEAN NOT NULL,
+    "TAGS" CHARACTER VARYING(500),
+    "CREATED_AT" BIGINT,
+    "UPDATED_AT" BIGINT NOT NULL
+);
+ALTER TABLE "PROJECTS" ADD CONSTRAINT "PK_NEW_PROJECTS" PRIMARY KEY("UUID");
+CREATE UNIQUE INDEX "UNIQ_PROJECTS_KEE" ON "PROJECTS"("KEE" NULLS FIRST);
+CREATE INDEX "IDX_QUALIFIER" ON "PROJECTS"("QUALIFIER" NULLS FIRST);
diff --git a/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v100/PopulateNclocForForProjectsTest/schema.sql b/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v100/PopulateNclocForForProjectsTest/schema.sql
new file mode 100644
index 00000000000..f69434e8266
--- /dev/null
+++ b/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v100/PopulateNclocForForProjectsTest/schema.sql
@@ -0,0 +1,67 @@
+CREATE TABLE "PROJECTS"(
+    "UUID" CHARACTER VARYING(40) NOT NULL,
+    "KEE" CHARACTER VARYING(400) NOT NULL,
+    "QUALIFIER" CHARACTER VARYING(10) NOT NULL,
+    "NAME" CHARACTER VARYING(2000),
+    "DESCRIPTION" CHARACTER VARYING(2000),
+    "PRIVATE" BOOLEAN NOT NULL,
+    "TAGS" CHARACTER VARYING(500),
+    "CREATED_AT" BIGINT,
+    "UPDATED_AT" BIGINT NOT NULL,
+    "NCLOC" BIGINT
+);
+ALTER TABLE "PROJECTS" ADD CONSTRAINT "PK_NEW_PROJECTS" PRIMARY KEY("UUID");
+CREATE UNIQUE INDEX "UNIQ_PROJECTS_KEE" ON "PROJECTS"("KEE" NULLS FIRST);
+CREATE INDEX "IDX_QUALIFIER" ON "PROJECTS"("QUALIFIER" NULLS FIRST);
+
+CREATE TABLE "PROJECT_BRANCHES"(
+    "UUID" CHARACTER VARYING(50) NOT NULL,
+    "PROJECT_UUID" CHARACTER VARYING(50) NOT NULL,
+    "KEE" CHARACTER VARYING(255) NOT NULL,
+    "BRANCH_TYPE" CHARACTER VARYING(12) NOT NULL,
+    "MERGE_BRANCH_UUID" CHARACTER VARYING(50),
+    "PULL_REQUEST_BINARY" BINARY LARGE OBJECT,
+    "MANUAL_BASELINE_ANALYSIS_UUID" CHARACTER VARYING(40),
+    "CREATED_AT" BIGINT NOT NULL,
+    "UPDATED_AT" BIGINT NOT NULL,
+    "EXCLUDE_FROM_PURGE" BOOLEAN DEFAULT FALSE NOT NULL,
+    "NEED_ISSUE_SYNC" BOOLEAN NOT NULL
+);
+ALTER TABLE "PROJECT_BRANCHES" ADD CONSTRAINT "PK_PROJECT_BRANCHES" PRIMARY KEY("UUID");
+CREATE UNIQUE INDEX "UNIQ_PROJECT_BRANCHES" ON "PROJECT_BRANCHES"("BRANCH_TYPE" NULLS FIRST, "PROJECT_UUID" NULLS FIRST, "KEE" NULLS FIRST);
+
+CREATE TABLE "LIVE_MEASURES"(
+    "UUID" CHARACTER VARYING(40) NOT NULL,
+    "PROJECT_UUID" CHARACTER VARYING(50) NOT NULL,
+    "COMPONENT_UUID" CHARACTER VARYING(50) NOT NULL,
+    "METRIC_UUID" CHARACTER VARYING(40) NOT NULL,
+    "VALUE" DOUBLE PRECISION,
+    "TEXT_VALUE" CHARACTER VARYING(4000),
+    "MEASURE_DATA" BINARY LARGE OBJECT,
+    "UPDATE_MARKER" CHARACTER VARYING(40),
+    "CREATED_AT" BIGINT NOT NULL,
+    "UPDATED_AT" BIGINT NOT NULL
+);
+ALTER TABLE "LIVE_MEASURES" ADD CONSTRAINT "PK_LIVE_MEASURES" PRIMARY KEY("UUID");
+CREATE INDEX "LIVE_MEASURES_PROJECT" ON "LIVE_MEASURES"("PROJECT_UUID" NULLS FIRST);
+CREATE UNIQUE INDEX "LIVE_MEASURES_COMPONENT" ON "LIVE_MEASURES"("COMPONENT_UUID" NULLS FIRST, "METRIC_UUID" NULLS FIRST);
+
+CREATE TABLE "METRICS"(
+    "UUID" CHARACTER VARYING(40) NOT NULL,
+    "NAME" CHARACTER VARYING(64) NOT NULL,
+    "DESCRIPTION" CHARACTER VARYING(255),
+    "DIRECTION" INTEGER DEFAULT 0 NOT NULL,
+    "DOMAIN" CHARACTER VARYING(64),
+    "SHORT_NAME" CHARACTER VARYING(64),
+    "QUALITATIVE" BOOLEAN DEFAULT FALSE NOT NULL,
+    "VAL_TYPE" CHARACTER VARYING(8),
+    "ENABLED" BOOLEAN DEFAULT TRUE,
+    "WORST_VALUE" DOUBLE PRECISION,
+    "BEST_VALUE" DOUBLE PRECISION,
+    "OPTIMIZED_BEST_VALUE" BOOLEAN,
+    "HIDDEN" BOOLEAN,
+    "DELETE_HISTORICAL_DATA" BOOLEAN,
+    "DECIMAL_SCALE" INTEGER
+);
+ALTER TABLE "METRICS" ADD CONSTRAINT "PK_METRICS" PRIMARY KEY("UUID");
+CREATE UNIQUE INDEX "METRICS_UNIQUE_NAME" ON "METRICS"("NAME" NULLS FIRST);