SONAR-23299 Add missing portfolios and views metrics and reindex project measure

9 files changed, 270 insertions, 7 deletions

diff --git a/server/sonar-db-migration/src/it/java/org/sonar/server/platform/db/migration/version/v108/DeleteSoftwareQualityRatingFromProjectMeasuresIT.java b/server/sonar-db-migration/src/it/java/org/sonar/server/platform/db/migration/version/v108/DeleteSoftwareQualityRatingFromProjectMeasuresIT.java
index 1bc0c0be4c7..e430671948e 100644
--- a/server/sonar-db-migration/src/it/java/org/sonar/server/platform/db/migration/version/v108/DeleteSoftwareQualityRatingFromProjectMeasuresIT.java
+++ b/server/sonar-db-migration/src/it/java/org/sonar/server/platform/db/migration/version/v108/DeleteSoftwareQualityRatingFromProjectMeasuresIT.java
@@ -23,8 +23,10 @@ import java.sql.SQLException;
 import java.util.Map;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
+import org.mockito.Mockito;
 import org.sonar.core.util.UuidFactoryImpl;
 import org.sonar.db.MigrationDbTester;
+import org.sonar.server.platform.db.migration.es.MigrationEsClient;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
@@ -33,7 +35,9 @@ class DeleteSoftwareQualityRatingFromProjectMeasuresIT {
   @RegisterExtension
   public final MigrationDbTester db = MigrationDbTester.createForMigrationStep(DeleteSoftwareQualityRatingFromProjectMeasures.class);
 
-  private final DeleteSoftwareQualityRatingFromProjectMeasures underTest = new DeleteSoftwareQualityRatingFromProjectMeasures(db.database());
+  private final MigrationEsClient migrationEsClient = Mockito.mock(MigrationEsClient.class);
+
+  private final DeleteSoftwareQualityRatingFromProjectMeasures underTest = new DeleteSoftwareQualityRatingFromProjectMeasures(db.database(), migrationEsClient);
 
   @Test
   void execute_whenMeasuresExists_shouldDeleteMeasures() throws SQLException {
@@ -48,6 +52,7 @@ class DeleteSoftwareQualityRatingFromProjectMeasuresIT {
     underTest.execute();
 
     assertThat(db.countSql("select count(1) from project_measures;")).isZero();
+    Mockito.verify(migrationEsClient, Mockito.times(1)).deleteIndexes("projectmeasures");
   }
 
   @Test
diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/es/MigrationEsClient.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/es/MigrationEsClient.java
new file mode 100644
index 00000000000..cb01d7188a5
--- /dev/null
+++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/es/MigrationEsClient.java
@@ -0,0 +1,32 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.platform.db.migration.es;
+
+/**
+ * This class is used to delete es index in order to trigger reindexation
+ * when a migration step has changed the data used by the index.
+ */
+public interface MigrationEsClient {
+
+  /**
+   * This method is re-entrant and does not fail if indexName or otherIndexNames do not exist
+   */
+  void deleteIndexes(String name, String... otherNames);
+}
diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/es/package-info.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/es/package-info.java
new file mode 100644
index 00000000000..9de338d20d3
--- /dev/null
+++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/es/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+@ParametersAreNonnullByDefault
+package org.sonar.server.platform.db.migration.es;
+
+import javax.annotation.ParametersAreNonnullByDefault;
diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v108/DeleteSoftwareQualityRatingFromProjectMeasures.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v108/DeleteSoftwareQualityRatingFromProjectMeasures.java
index 92dfcaff2e8..8dd32588f1a 100644
--- a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v108/DeleteSoftwareQualityRatingFromProjectMeasures.java
+++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v108/DeleteSoftwareQualityRatingFromProjectMeasures.java
@@ -24,6 +24,7 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import org.sonar.core.metric.SoftwareQualitiesMetrics;
 import org.sonar.db.Database;
+import org.sonar.server.platform.db.migration.es.MigrationEsClient;
 import org.sonar.server.platform.db.migration.step.DataChange;
 import org.sonar.server.platform.db.migration.step.MassUpdate;
 
@@ -44,7 +45,23 @@ public class DeleteSoftwareQualityRatingFromProjectMeasures extends DataChange {
     SoftwareQualitiesMetrics.SOFTWARE_QUALITY_RELIABILITY_REMEDIATION_EFFORT_KEY,
     SoftwareQualitiesMetrics.NEW_SOFTWARE_QUALITY_RELIABILITY_REMEDIATION_EFFORT_KEY,
     SoftwareQualitiesMetrics.SOFTWARE_QUALITY_MAINTAINABILITY_DEBT_RATIO_KEY,
-    SoftwareQualitiesMetrics.NEW_SOFTWARE_QUALITY_MAINTAINABILITY_DEBT_RATIO_KEY);
+    SoftwareQualitiesMetrics.NEW_SOFTWARE_QUALITY_MAINTAINABILITY_DEBT_RATIO_KEY,
+
+    // Portfolios
+    "software_quality_reliability_rating_distribution",
+    "software_quality_security_rating_distribution",
+    "software_quality_maintainability_rating_distribution",
+    "new_software_quality_maintainability_rating_distribution",
+    "new_software_quality_reliability_rating_distribution",
+    "new_software_quality_security_rating_distribution",
+
+    // Views
+    "last_change_on_software_quality_security_rating",
+    "last_change_on_software_quality_reliability_rating",
+    "last_change_on_software_quality_maintainability_rating",
+    "software_quality_security_rating_effort",
+    "software_quality_reliability_rating_effort",
+    "software_quality_maintainability_rating_effort");
 
   private static final String SELECT_QUERY = """
     select pm.uuid from project_measures pm
@@ -52,8 +69,11 @@ public class DeleteSoftwareQualityRatingFromProjectMeasures extends DataChange {
      where m.name in (%s)
     """.formatted(SOFTWARE_QUALITY_METRICS_TO_DELETE.stream().map(s -> "'" + s + "'").collect(Collectors.joining(",")));
 
-  public DeleteSoftwareQualityRatingFromProjectMeasures(Database db) {
+  private final MigrationEsClient migrationEsClient;
+
+  public DeleteSoftwareQualityRatingFromProjectMeasures(Database db, MigrationEsClient migrationEsClient) {
     super(db);
+    this.migrationEsClient = migrationEsClient;
   }
 
   @Override
@@ -66,5 +86,8 @@ public class DeleteSoftwareQualityRatingFromProjectMeasures extends DataChange {
       update.setString(1, row.getString(1));
       return true;
     });
+
+    // Reindexation of project measures is required to align with removed values
+    migrationEsClient.deleteIndexes("projectmeasures");
   }
 }
diff --git a/server/sonar-db-migration/src/testFixtures/java/org/sonar/db/NoOpMigrationEsClient.java b/server/sonar-db-migration/src/testFixtures/java/org/sonar/db/NoOpMigrationEsClient.java
new file mode 100644
index 00000000000..581ce5f0c0f
--- /dev/null
+++ b/server/sonar-db-migration/src/testFixtures/java/org/sonar/db/NoOpMigrationEsClient.java
@@ -0,0 +1,29 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.db;
+
+import org.sonar.server.platform.db.migration.es.MigrationEsClient;
+
+public class NoOpMigrationEsClient implements MigrationEsClient {
+  @Override
+  public void deleteIndexes(String name, String... otherNames) {
+
+  }
+}
diff --git a/server/sonar-db-migration/src/testFixtures/java/org/sonar/db/SQDatabase.java b/server/sonar-db-migration/src/testFixtures/java/org/sonar/db/SQDatabase.java
index 29d3c8419d9..523deaee66e 100644
--- a/server/sonar-db-migration/src/testFixtures/java/org/sonar/db/SQDatabase.java
+++ b/server/sonar-db-migration/src/testFixtures/java/org/sonar/db/SQDatabase.java
@@ -186,6 +186,7 @@ public class SQDatabase extends DefaultDatabase {
     container.add(UuidFactoryFast.getInstance());
     container.add(System2.INSTANCE);
     container.add(MapSettings.class);
+    container.add(NoOpMigrationEsClient.class);
 
     container.startComponents();
     MigrationContainer migrationContainer = new MigrationContainerImpl(container, H2StepExecutor.class);
diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/es/MigrationEsClientImpl.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/es/MigrationEsClientImpl.java
new file mode 100644
index 00000000000..eb8c4e2ae8f
--- /dev/null
+++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/es/MigrationEsClientImpl.java
@@ -0,0 +1,57 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.es;
+
+import java.util.Arrays;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.client.indices.GetIndexRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonar.server.platform.db.migration.es.MigrationEsClient;
+
+public class MigrationEsClientImpl implements MigrationEsClient {
+  private static final Logger LOG = LoggerFactory.getLogger(MigrationEsClientImpl.class);
+  private final EsClient client;
+
+  public MigrationEsClientImpl(EsClient client) {
+    this.client = client;
+  }
+
+  @Override
+  public void deleteIndexes(String name, String... otherNames) {
+    String[] indices = client.getIndex(new GetIndexRequest("_all")).getIndices();
+    Set<String> existingIndices = Arrays.stream(indices).collect(Collectors.toSet());
+    String[] toDelete = Stream.concat(Stream.of(name), Arrays.stream(otherNames))
+      .distinct()
+      .filter(existingIndices::contains)
+      .toArray(String[]::new);
+    if (toDelete.length > 0) {
+      deleteIndex(toDelete);
+    }
+  }
+
+  private void deleteIndex(String... indices) {
+    LOG.info("Drop Elasticsearch indices [{}]", String.join(",", indices));
+    client.deleteIndex(new DeleteIndexRequest(indices));
+  }
+}
diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java
new file mode 100644
index 00000000000..4b42b6d6c47
--- /dev/null
+++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java
@@ -0,0 +1,91 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.es;
+
+import java.util.Iterator;
+import org.elasticsearch.client.indices.GetMappingsRequest;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.slf4j.event.Level;
+import org.sonar.api.config.internal.MapSettings;
+import org.sonar.api.testfixtures.log.LogTesterJUnit5;
+import org.sonar.server.platform.db.migration.es.MigrationEsClient;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder;
+
+class MigrationEsClientImplTest {
+  @RegisterExtension
+  public LogTesterJUnit5 logTester = new LogTesterJUnit5();
+
+  @RegisterExtension
+  public EsTester es = EsTester.createCustom(
+    new SimpleIndexDefinition("as"),
+    new SimpleIndexDefinition("bs"),
+    new SimpleIndexDefinition("cs"));
+
+  private final MigrationEsClient underTest = new MigrationEsClientImpl(es.client());
+
+  @Test
+  void delete_existing_index() {
+    underTest.deleteIndexes("as");
+
+    assertThat(loadExistingIndices())
+      .toIterable()
+      .doesNotContain("as")
+      .contains("bs", "cs");
+    assertThat(logTester.logs(Level.INFO))
+      .contains("Drop Elasticsearch indices [as]");
+  }
+
+  @Test
+  void delete_index_that_does_not_exist() {
+    underTest.deleteIndexes("as", "xxx", "cs");
+
+    assertThat(loadExistingIndices())
+      .toIterable()
+      .doesNotContain("as", "cs")
+      .contains("bs");
+    assertThat(logTester.logs(Level.INFO))
+      .contains("Drop Elasticsearch indices [as,cs]")
+      .doesNotContain("Drop Elasticsearch indices [xxx]");
+  }
+
+  private Iterator<String> loadExistingIndices() {
+    return es.client().getMapping(new GetMappingsRequest()).mappings().keySet().iterator();
+  }
+
+  private static class SimpleIndexDefinition implements IndexDefinition {
+    private final String indexName;
+
+    public SimpleIndexDefinition(String indexName) {
+      this.indexName = indexName;
+    }
+
+    @Override
+    public void define(IndexDefinitionContext context) {
+      IndexType.IndexMainType mainType = IndexType.main(Index.simple(indexName), indexName.substring(1));
+      context.create(
+        mainType.getIndex(),
+        newBuilder(new MapSettings().asConfig()).build())
+        .createTypeMapping(mainType);
+    }
+  }
+}
diff --git a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel2.java b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel2.java
index 3bb99f3731f..2e9be43668f 100644
--- a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel2.java
+++ b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel2.java
@@ -24,6 +24,7 @@ import org.sonar.core.extension.CoreExtensionsInstaller;
 import org.sonar.core.platform.PluginClassLoader;
 import org.sonar.core.platform.PluginClassloaderFactory;
 import org.sonar.core.platform.SpringComponentContainer;
+import org.sonar.server.es.MigrationEsClientImpl;
 import org.sonar.server.l18n.ServerI18n;
 import org.sonar.server.platform.DatabaseServerCompatibility;
 import org.sonar.server.platform.DefaultServerUpgradeStatus;
@@ -36,10 +37,6 @@ import org.sonar.server.platform.db.migration.DatabaseMigrationStateImpl;
 import org.sonar.server.platform.db.migration.MigrationConfigurationModule;
 import org.sonar.server.platform.db.migration.charset.DatabaseCharsetChecker;
 import org.sonar.server.platform.db.migration.version.DatabaseVersion;
-import org.sonar.server.telemetry.TelemetryDbMigrationStepDurationProvider;
-import org.sonar.server.telemetry.TelemetryDbMigrationSuccessProvider;
-import org.sonar.server.telemetry.TelemetryDbMigrationStepsProvider;
-import org.sonar.server.telemetry.TelemetryDbMigrationTotalTimeProvider;
 import org.sonar.server.platform.web.WebPagesCache;
 import org.sonar.server.plugins.InstalledPluginReferentialFactory;
 import org.sonar.server.plugins.PluginJarLoader;
@@ -47,6 +44,10 @@ import org.sonar.server.plugins.ServerPluginJarExploder;
 import org.sonar.server.plugins.ServerPluginManager;
 import org.sonar.server.plugins.ServerPluginRepository;
 import org.sonar.server.plugins.WebServerExtensionInstaller;
+import org.sonar.server.telemetry.TelemetryDbMigrationStepDurationProvider;
+import org.sonar.server.telemetry.TelemetryDbMigrationStepsProvider;
+import org.sonar.server.telemetry.TelemetryDbMigrationSuccessProvider;
+import org.sonar.server.telemetry.TelemetryDbMigrationTotalTimeProvider;
 
 import static org.sonar.core.extension.CoreExtensionsInstaller.noAdditionalSideFilter;
 import static org.sonar.core.extension.PlatformLevelPredicates.hasPlatformLevel;
@@ -60,6 +61,7 @@ public class PlatformLevel2 extends PlatformLevel {
   protected void configureLevel() {
     add(
       new MigrationConfigurationModule(),
+      MigrationEsClientImpl.class,
       DatabaseVersion.class,
       DatabaseServerCompatibility.class,