SONAR-20058 improve webhook purge mechanism

server/sonar-db-dao/src/it/java/org/sonar/db/webhook/WebhookDeliveryDaoIT.java

@@ -208,32 +208,24 @@ public class WebhookDeliveryDaoIT {
   @Test
   public void deleteComponentBeforeDate_deletes_rows_before_date() {
     underTest.insert(dbSession, WebhookDeliveryTesting.newDto("DELIVERY_1", "WEBHOOK_UUID_1", "PROJECT_1", "TASK_1").setCreatedAt(1_000_000L));
-    underTest.insert(dbSession, WebhookDeliveryTesting.newDto("DELIVERY_2", "WEBHOOK_UUID_1", "PROJECT_1", "TASK_2").setCreatedAt(2_000_000L));
-    underTest.insert(dbSession, WebhookDeliveryTesting.newDto("DELIVERY_3", "WEBHOOK_UUID_1", "PROJECT_2", "TASK_3").setCreatedAt(1_000_000L));
+    underTest.insert(dbSession, WebhookDeliveryTesting.newDto("DELIVERY_2", "WEBHOOK_UUID_2", "PROJECT_1", "TASK_2").setCreatedAt(2_000_000L));
+    underTest.insert(dbSession, WebhookDeliveryTesting.newDto("DELIVERY_3", "WEBHOOK_UUID_3", "PROJECT_2", "TASK_3").setCreatedAt(1_000_000L));
+    underTest.insert(dbSession, WebhookDeliveryTesting.newDto("DELIVERY_4", "WEBHOOK_UUID_4", "PROJECT_3", "TASK_4").setCreatedAt(2_000_000L));
 
     // should delete the old delivery on PROJECT_1 and keep the one of PROJECT_2
-    underTest.deleteProjectBeforeDate(dbSession, "PROJECT_1", 1_500_000L);
+    underTest.deleteAllBeforeDate(dbSession, 1_500_000L);
 
     List<Map<String, Object>> uuids = dbTester.select(dbSession, "select uuid as \"uuid\" from webhook_deliveries");
-    assertThat(uuids).extracting(column -> column.get("uuid")).containsOnly("DELIVERY_2", "DELIVERY_3");
+    assertThat(uuids).extracting(column -> column.get("uuid")).containsOnly("DELIVERY_2", "DELIVERY_4");
   }
 
   @Test
   public void deleteComponentBeforeDate_does_nothing_on_empty_table() {
-    underTest.deleteProjectBeforeDate(dbSession, "PROJECT_1", 1_500_000L);
+    underTest.deleteAllBeforeDate(dbSession, 1_500_000L);
 
     assertThat(dbTester.countRowsOfTable(dbSession, "webhook_deliveries")).isZero();
   }
 
-  @Test
-  public void deleteComponentBeforeDate_does_nothing_on_invalid_uuid() {
-    underTest.insert(dbSession, WebhookDeliveryTesting.newDto("DELIVERY_1", "WEBHOOK_UUID_1", "PROJECT_1", "TASK_1").setCreatedAt(1_000_000L));
-
-    underTest.deleteProjectBeforeDate(dbSession, "PROJECT_2", 1_500_000L);
-
-    assertThat(dbTester.countRowsOfTable(dbSession, "webhook_deliveries")).isOne();
-  }
-
   private void verifyMandatoryFields(WebhookDeliveryDto expected, WebhookDeliveryDto actual) {
     assertThat(actual.getUuid()).isEqualTo(expected.getUuid());
     assertThat(actual.getProjectUuid()).isEqualTo(expected.getProjectUuid());

server/sonar-db-dao/src/main/java/org/sonar/db/webhook/WebhookDeliveryDao.java

@@ -72,8 +72,8 @@ public class WebhookDeliveryDao implements Dao {
     mapper(dbSession).insert(dto);
   }
 
-  public void deleteProjectBeforeDate(DbSession dbSession, String projectUuid, long beforeDate) {
-    mapper(dbSession).deleteProjectBeforeDate(projectUuid, beforeDate);
+  public void deleteAllBeforeDate(DbSession dbSession, long beforeDate) {
+    mapper(dbSession).deleteAllBeforeDate(beforeDate);
   }
 
   public Map<String, WebhookDeliveryLiteDto> selectLatestDeliveries(DbSession dbSession, List<WebhookDto> webhooks) {

server/sonar-db-dao/src/main/java/org/sonar/db/webhook/WebhookDeliveryMapper.java

@@ -43,7 +43,7 @@ public interface WebhookDeliveryMapper {
 
   void insert(WebhookDeliveryDto dto);
 
-  void deleteProjectBeforeDate(@Param("projectUuid") String projectUuid, @Param("beforeDate") long beforeDate);
+  void deleteAllBeforeDate(@Param("beforeDate") long beforeDate);
 
   void deleteByWebhookUuid(@Param("webhookUuid") String webhookUuid);
 }

server/sonar-db-dao/src/main/resources/org/sonar/db/webhook/WebhookDeliveryMapper.xml

@@ -108,10 +108,9 @@
     where webhook_uuid = #{webhookUuid,jdbcType=VARCHAR}
   </select>
 
-  <delete id="deleteProjectBeforeDate" parameterType="map">
+  <delete id="deleteAllBeforeDate" parameterType="map">
     delete from webhook_deliveries
     where
-    project_uuid = #{projectUuid,jdbcType=VARCHAR} and
     created_at &lt; #{beforeDate,jdbcType=BIGINT}
   </delete>
 </mapper>

server/sonar-db-dao/src/schema/schema-sq.ddl

@@ -1105,6 +1105,7 @@ ALTER TABLE "WEBHOOK_DELIVERIES" ADD CONSTRAINT "PK_WEBHOOK_DELIVERIES" PRIMARY
 CREATE INDEX "WD_WEBHOOK_UUID_CREATED_AT" ON "WEBHOOK_DELIVERIES"("WEBHOOK_UUID" NULLS FIRST, "CREATED_AT" NULLS FIRST);
 CREATE INDEX "WD_PROJECT_UUID_CREATED_AT" ON "WEBHOOK_DELIVERIES"("PROJECT_UUID" NULLS FIRST, "CREATED_AT" NULLS FIRST);
 CREATE INDEX "WD_CE_TASK_UUID_CREATED_AT" ON "WEBHOOK_DELIVERIES"("CE_TASK_UUID" NULLS FIRST, "CREATED_AT" NULLS FIRST);
+CREATE INDEX "WD_CREATED_AT" ON "WEBHOOK_DELIVERIES"("CREATED_AT" NULLS FIRST);
 
 CREATE TABLE "WEBHOOKS"(
     "UUID" CHARACTER VARYING(40) NOT NULL,

server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v102/CreateIndexCreatedAtInWebhookDeliveries.java

@@ -0,0 +1,29 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.platform.db.migration.version.v102;
+
+import org.sonar.db.Database;
+import org.sonar.server.platform.db.migration.step.CreateIndexOnColumns;
+
+public class CreateIndexCreatedAtInWebhookDeliveries extends CreateIndexOnColumns {
+  protected CreateIndexCreatedAtInWebhookDeliveries(Database db) {
+    super(db, "webhook_deliveries", "wd", false, "created_at");
+  }
+}

server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v102/DbVersion102.java

@@ -109,6 +109,8 @@ public class DbVersion102 implements DbVersion {
       .add(10_2_050, "Create index 'wd_project_uuid_created_at' in 'webhook_deliveries'", CreateIndexProjectUuidCreatedAtInWebhookDeliveries.class)
       .add(10_2_051, "Drop index 'ce_task_uuid' in 'webhook_deliveries'", DropIndexTaskUuidInWebhookDeliveries.class)
       .add(10_2_052, "Create index 'wd_task_uuid_created_at' in 'webhook_deliveries'", CreateIndexTaskUuidCreatedAtInWebhookDeliveries.class)
+      .add(10_2_053, "Create index 'wd_created_at' in 'webhook_deliveries'", CreateIndexCreatedAtInWebhookDeliveries.class)
+
     ;
   }
 }

server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v102/CreateIndexCreatedAtInWebhookDeliveriesTest.java

@@ -0,0 +1,55 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.platform.db.migration.version.v102;
+
+import java.sql.SQLException;
+import org.junit.Rule;
+import org.junit.Test;
+import org.sonar.db.CoreDbTester;
+import org.sonar.server.platform.db.migration.step.DdlChange;
+
+public class CreateIndexCreatedAtInWebhookDeliveriesTest {
+
+  public static final String TABLE_NAME = "webhook_deliveries";
+  public static final String INDEX_NAME = "wd_created_at";
+  public static final String EXPECTED_COLUMN = "created_at";
+  @Rule
+  public final CoreDbTester db = CoreDbTester.createForSchema(CreateIndexCreatedAtInWebhookDeliveriesTest.class, "schema.sql");
+
+  private final DdlChange createIndex = new CreateIndexCreatedAtInWebhookDeliveries(db.database());
+
+  @Test
+  public void migration_should_create_index() throws SQLException {
+    db.assertIndexDoesNotExist(TABLE_NAME, INDEX_NAME);
+
+    createIndex.execute();
+
+    db.assertIndex(TABLE_NAME, INDEX_NAME, EXPECTED_COLUMN);
+  }
+
+  @Test
+  public void migration_should_be_reentrant() throws SQLException {
+    createIndex.execute();
+    createIndex.execute();
+
+    db.assertIndex(TABLE_NAME, INDEX_NAME, EXPECTED_COLUMN);
+  }
+
+}

server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v102/CreateIndexCreatedAtInWebhookDeliveriesTest/schema.sql

@@ -0,0 +1,19 @@
+CREATE TABLE "WEBHOOK_DELIVERIES"(
+    "UUID" CHARACTER VARYING(40) NOT NULL,
+    "WEBHOOK_UUID" CHARACTER VARYING(40) NOT NULL,
+    "PROJECT_UUID" CHARACTER VARYING(40) NOT NULL,
+    "CE_TASK_UUID" CHARACTER VARYING(40),
+    "ANALYSIS_UUID" CHARACTER VARYING(40),
+    "NAME" CHARACTER VARYING(100) NOT NULL,
+    "URL" CHARACTER VARYING(2000) NOT NULL,
+    "SUCCESS" BOOLEAN NOT NULL,
+    "HTTP_STATUS" INTEGER,
+    "DURATION_MS" BIGINT NOT NULL,
+    "PAYLOAD" CHARACTER LARGE OBJECT NOT NULL,
+    "ERROR_STACKTRACE" CHARACTER LARGE OBJECT,
+    "CREATED_AT" BIGINT NOT NULL
+);
+ALTER TABLE "WEBHOOK_DELIVERIES" ADD CONSTRAINT "PK_WEBHOOK_DELIVERIES" PRIMARY KEY("UUID");
+CREATE INDEX "WD_WEBHOOK_UUID_CREATED_AT" ON "WEBHOOK_DELIVERIES"("WEBHOOK_UUID", "CREATED_AT" NULLS FIRST);
+CREATE INDEX "WD_PROJECT_UUID_CREATED_AT" ON "WEBHOOK_DELIVERIES"("PROJECT_UUID", "CREATED_AT" NULLS FIRST);
+CREATE INDEX "WD_CE_TASK_UUID_CREATED_AT" ON "WEBHOOK_DELIVERIES"("CE_TASK_UUID", "CREATED_AT" NULLS FIRST);

server/sonar-server-common/src/it/java/org/sonar/server/webhook/AsynchronousWebHooksImplIT.java

@@ -78,7 +78,7 @@ public class AsynchronousWebHooksImplIT {
 
     assertThat(caller.countSent()).isEqualTo(2);
     verify(deliveryStorage, times(2)).persist(any(WebhookDelivery.class));
-    verify(deliveryStorage).purge(project.getUuid());
+    verify(deliveryStorage).purge();
   }
 
   private static class RecordingAsyncExecution implements AsyncExecution {

server/sonar-server-common/src/it/java/org/sonar/server/webhook/SynchronousWebHooksImplIT.java

@@ -130,7 +130,7 @@ public class SynchronousWebHooksImplIT {
     assertThat(logTester.logs(DEBUG)).contains("Sent webhook 'First' | url=http://url1 | time=1234ms | status=200");
     assertThat(logTester.logs(DEBUG)).contains("Failed to send webhook 'Second' | url=http://url2 | message=Fail to connect");
     verify(deliveryStorage, times(2)).persist(any(WebhookDelivery.class));
-    verify(deliveryStorage).purge(projectDto.getUuid());
+    verify(deliveryStorage).purge();
     verifyLogStatistics(2, 0);
   }
 
@@ -151,7 +151,7 @@ public class SynchronousWebHooksImplIT {
     assertThat(caller.countSent()).isOne();
     assertThat(logTester.logs(DEBUG)).contains("Sent webhook 'First' | url=http://url1 | time=1234ms | status=200");
     verify(deliveryStorage).persist(any(WebhookDelivery.class));
-    verify(deliveryStorage).purge(projectDto.getUuid());
+    verify(deliveryStorage).purge();
     verifyLogStatistics(0, 1);
   }
 
@@ -180,7 +180,7 @@ public class SynchronousWebHooksImplIT {
       .contains("Sent webhook '4Fourth' | url=http://url4 | time=5678ms | status=200")
       .contains("Sent webhook '5Fifth' | url=http://url5 | time=9256ms | status=200");
     verify(deliveryStorage, times(5)).persist(any(WebhookDelivery.class));
-    verify(deliveryStorage).purge(projectDto.getUuid());
+    verify(deliveryStorage).purge();
     verifyLogStatistics(3, 2);
   }
 

server/sonar-server-common/src/it/java/org/sonar/server/webhook/WebhookDeliveryStorageIT.java

@@ -89,17 +89,17 @@ public class WebhookDeliveryStorageIT {
   }
 
   @Test
-  public void purge_deletes_records_older_than_one_month_on_the_project() {
+  public void purge_deletes_records_older_than_one_month() {
     when(system.now()).thenReturn(NOW);
     dbClient.webhookDeliveryDao().insert(dbSession, newDto("D1", "PROJECT_1", TWO_MONTHS_AGO));
     dbClient.webhookDeliveryDao().insert(dbSession, newDto("D2", "PROJECT_1", TWO_WEEKS_AGO));
     dbClient.webhookDeliveryDao().insert(dbSession, newDto("D3", "PROJECT_2", TWO_MONTHS_AGO));
+    dbClient.webhookDeliveryDao().insert(dbSession, newDto("D4", "PROJECT_2", TWO_WEEKS_AGO));
     dbSession.commit();
 
-    underTest.purge("PROJECT_1");
+    underTest.purge();
 
-    // do not purge another project PROJECT_2
-    assertThat(selectAllDeliveryUuids(dbTester, dbSession)).containsOnly("D2", "D3");
+    assertThat(selectAllDeliveryUuids(dbTester, dbSession)).containsOnly("D2", "D4");
   }
 
   @Test

server/sonar-server-common/src/main/java/org/sonar/server/webhook/WebHooksImpl.java

@@ -116,7 +116,7 @@ public class WebHooksImpl implements WebHooks {
       log(delivery);
       deliveryStorage.persist(delivery);
     }));
-    asyncExecution.addToQueue(() -> deliveryStorage.purge(analysis.projectUuid()));
+    asyncExecution.addToQueue(deliveryStorage::purge);
   }
 
   private static void log(WebhookDelivery delivery) {

server/sonar-server-common/src/main/java/org/sonar/server/webhook/WebhookDeliveryStorage.java

@@ -56,10 +56,10 @@ public class WebhookDeliveryStorage {
     }
   }
 
-  public void purge(String projectUuid) {
+  public void purge() {
     long beforeDate = system.now() - ALIVE_DELAY_MS;
     try (DbSession dbSession = dbClient.openSession(false)) {
-      dbClient.webhookDeliveryDao().deleteProjectBeforeDate(dbSession, projectUuid, beforeDate);
+      dbClient.webhookDeliveryDao().deleteAllBeforeDate(dbSession, beforeDate);
       dbSession.commit();
     }
   }