SONAR-17444 Analyzers's cache should expire after 7 days

9 files changed, 351 insertions, 1 deletions

diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/analysis/cache/cleaning/AnalysisCacheCleaningExecutorService.java b/server/sonar-ce/src/main/java/org/sonar/ce/analysis/cache/cleaning/AnalysisCacheCleaningExecutorService.java
new file mode 100644
index 00000000000..ce1a7265056
--- /dev/null
+++ b/server/sonar-ce/src/main/java/org/sonar/ce/analysis/cache/cleaning/AnalysisCacheCleaningExecutorService.java
@@ -0,0 +1,25 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.ce.analysis.cache.cleaning;
+
+import java.util.concurrent.ScheduledExecutorService;
+
+public interface AnalysisCacheCleaningExecutorService extends ScheduledExecutorService {
+}
diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/analysis/cache/cleaning/AnalysisCacheCleaningExecutorServiceImpl.java b/server/sonar-ce/src/main/java/org/sonar/ce/analysis/cache/cleaning/AnalysisCacheCleaningExecutorServiceImpl.java
new file mode 100644
index 00000000000..fa6f33d225f
--- /dev/null
+++ b/server/sonar-ce/src/main/java/org/sonar/ce/analysis/cache/cleaning/AnalysisCacheCleaningExecutorServiceImpl.java
@@ -0,0 +1,37 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.ce.analysis.cache.cleaning;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import org.sonar.server.util.AbstractStoppableScheduledExecutorServiceImpl;
+
+public class AnalysisCacheCleaningExecutorServiceImpl extends AbstractStoppableScheduledExecutorServiceImpl<ScheduledExecutorService>
+  implements AnalysisCacheCleaningExecutorService {
+
+  public AnalysisCacheCleaningExecutorServiceImpl() {
+    super(Executors.newSingleThreadScheduledExecutor(
+      new ThreadFactoryBuilder()
+        .setDaemon(true)
+        .setNameFormat("Analysis_cache_cleaning-%d")
+        .build()));
+  }
+}
diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/analysis/cache/cleaning/AnalysisCacheCleaningModule.java b/server/sonar-ce/src/main/java/org/sonar/ce/analysis/cache/cleaning/AnalysisCacheCleaningModule.java
new file mode 100644
index 00000000000..26b598fb063
--- /dev/null
+++ b/server/sonar-ce/src/main/java/org/sonar/ce/analysis/cache/cleaning/AnalysisCacheCleaningModule.java
@@ -0,0 +1,31 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.ce.analysis.cache.cleaning;
+
+import org.sonar.core.platform.Module;
+
+public class AnalysisCacheCleaningModule extends Module {
+  @Override protected void configureModule() {
+    add(
+      AnalysisCacheCleaningExecutorServiceImpl.class,
+      AnalysisCacheCleaningSchedulerImpl.class
+    );
+  }
+}
diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/analysis/cache/cleaning/AnalysisCacheCleaningScheduler.java b/server/sonar-ce/src/main/java/org/sonar/ce/analysis/cache/cleaning/AnalysisCacheCleaningScheduler.java
new file mode 100644
index 00000000000..91ef19a13f5
--- /dev/null
+++ b/server/sonar-ce/src/main/java/org/sonar/ce/analysis/cache/cleaning/AnalysisCacheCleaningScheduler.java
@@ -0,0 +1,25 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.ce.analysis.cache.cleaning;
+
+import org.sonar.api.platform.ServerStartHandler;
+
+public interface AnalysisCacheCleaningScheduler extends ServerStartHandler {
+}
diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/analysis/cache/cleaning/AnalysisCacheCleaningSchedulerImpl.java b/server/sonar-ce/src/main/java/org/sonar/ce/analysis/cache/cleaning/AnalysisCacheCleaningSchedulerImpl.java
new file mode 100644
index 00000000000..47438fc821e
--- /dev/null
+++ b/server/sonar-ce/src/main/java/org/sonar/ce/analysis/cache/cleaning/AnalysisCacheCleaningSchedulerImpl.java
@@ -0,0 +1,55 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.ce.analysis.cache.cleaning;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.time.Duration;
+import java.time.LocalDateTime;
+import org.sonar.api.platform.Server;
+import org.sonar.db.DbClient;
+
+import static java.util.concurrent.TimeUnit.DAYS;
+import static java.util.concurrent.TimeUnit.SECONDS;
+
+public class AnalysisCacheCleaningSchedulerImpl implements AnalysisCacheCleaningScheduler {
+  private final AnalysisCacheCleaningExecutorService executorService;
+  private final DbClient dbClient;
+
+  public AnalysisCacheCleaningSchedulerImpl(AnalysisCacheCleaningExecutorService executorService, DbClient dbClient) {
+    this.executorService = executorService;
+    this.dbClient = dbClient;
+  }
+
+  @Override public void onServerStart(Server server) {
+    LocalDateTime now = LocalDateTime.now();
+    // schedule run at midnight everyday
+    LocalDateTime nextRun = now.plusDays(1).withHour(0).withMinute(0).withSecond(0);
+    long initialDelay = Duration.between(now, nextRun).getSeconds();
+    executorService.scheduleAtFixedRate(this::clean, initialDelay, DAYS.toSeconds(1), SECONDS);
+  }
+
+  @VisibleForTesting
+  void clean() {
+    try (var dbSession = dbClient.openSession(false)) {
+      dbClient.scannerAnalysisCacheDao().cleanOlderThan7Days(dbSession);
+      dbSession.commit();
+    }
+  }
+}
diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java b/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java
index fd41e64a6c4..8e5708e2589 100644
--- a/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java
+++ b/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java
@@ -45,6 +45,7 @@ import org.sonar.ce.CeHttpModule;
 import org.sonar.ce.CeQueueModule;
 import org.sonar.ce.CeTaskCommonsModule;
 import org.sonar.ce.StandaloneCeDistributedInformation;
+import org.sonar.ce.analysis.cache.cleaning.AnalysisCacheCleaningModule;
 import org.sonar.ce.async.SynchronousAsyncExecution;
 import org.sonar.ce.cleaning.CeCleaningModule;
 import org.sonar.ce.db.ReadOnlyPropertiesDao;
@@ -446,7 +447,9 @@ public class ComputeEngineContainerImpl implements ComputeEngineContainer {
       new WebhookModule(),
 
       QualityGateFinder.class,
-      QualityGateEvaluatorImpl.class
+      QualityGateEvaluatorImpl.class,
+
+      new AnalysisCacheCleaningModule()
 
     );
 
diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/analysis/cache/cleaning/AnalysisCacheCleaningExecutorServiceImplTest.java b/server/sonar-ce/src/test/java/org/sonar/ce/analysis/cache/cleaning/AnalysisCacheCleaningExecutorServiceImplTest.java
new file mode 100644
index 00000000000..3209b67ac19
--- /dev/null
+++ b/server/sonar-ce/src/test/java/org/sonar/ce/analysis/cache/cleaning/AnalysisCacheCleaningExecutorServiceImplTest.java
@@ -0,0 +1,35 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.ce.analysis.cache.cleaning;
+
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class AnalysisCacheCleaningExecutorServiceImplTest {
+
+  @Test
+  public void constructor_createsExecutorDelegateThatIsReadyToAct() {
+    AnalysisCacheCleaningExecutorServiceImpl underTest = new AnalysisCacheCleaningExecutorServiceImpl();
+
+    assertThat(underTest.isShutdown()).isFalse();
+    assertThat(underTest.isTerminated()).isFalse();
+  }
+}
diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/analysis/cache/cleaning/AnalysisCacheCleaningModuleTest.java b/server/sonar-ce/src/test/java/org/sonar/ce/analysis/cache/cleaning/AnalysisCacheCleaningModuleTest.java
new file mode 100644
index 00000000000..08cf9a7f518
--- /dev/null
+++ b/server/sonar-ce/src/test/java/org/sonar/ce/analysis/cache/cleaning/AnalysisCacheCleaningModuleTest.java
@@ -0,0 +1,34 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.ce.analysis.cache.cleaning;
+
+import org.junit.Test;
+import org.sonar.core.platform.ListContainer;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class AnalysisCacheCleaningModuleTest {
+  @Test
+  public void verify_count_of_added_components() {
+    ListContainer container = new ListContainer();
+    new AnalysisCacheCleaningModule().configure(container);
+    assertThat(container.getAddedObjects()).hasSize(2);
+  }
+}
diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/analysis/cache/cleaning/AnalysisCacheCleaningSchedulerImplTest.java b/server/sonar-ce/src/test/java/org/sonar/ce/analysis/cache/cleaning/AnalysisCacheCleaningSchedulerImplTest.java
new file mode 100644
index 00000000000..9b65667416c
--- /dev/null
+++ b/server/sonar-ce/src/test/java/org/sonar/ce/analysis/cache/cleaning/AnalysisCacheCleaningSchedulerImplTest.java
@@ -0,0 +1,105 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.ce.analysis.cache.cleaning;
+
+import java.io.ByteArrayInputStream;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import org.junit.Rule;
+import org.junit.Test;
+import org.sonar.api.platform.Server;
+import org.sonar.api.utils.System2;
+import org.sonar.core.util.SequenceUuidFactory;
+import org.sonar.core.util.UuidFactory;
+import org.sonar.db.DbSession;
+import org.sonar.db.DbTester;
+import org.sonar.db.component.SnapshotDto;
+import org.sonar.db.scannercache.ScannerAnalysisCacheDao;
+
+import static java.util.concurrent.TimeUnit.DAYS;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class AnalysisCacheCleaningSchedulerImplTest {
+  private System2 system2 = mock(System2.class);
+  private final static UuidFactory uuidFactory = new SequenceUuidFactory();
+  @Rule
+  public DbTester dbTester = DbTester.create(system2);
+  private DbSession dbSession = dbTester.getSession();
+  private ScannerAnalysisCacheDao scannerAnalysisCacheDao = dbTester.getDbClient().scannerAnalysisCacheDao();
+
+  AnalysisCacheCleaningExecutorService executorService = mock(AnalysisCacheCleaningExecutorService.class);
+
+  AnalysisCacheCleaningSchedulerImpl underTest = new AnalysisCacheCleaningSchedulerImpl(executorService, dbTester.getDbClient());
+
+  @Test
+  public void startSchedulingOnServerStart() {
+    underTest.onServerStart(mock(Server.class));
+    verify(executorService, times(1)).scheduleAtFixedRate(any(Runnable.class), anyLong(), eq(DAYS.toSeconds(1)), eq(SECONDS));
+  }
+
+  @Test
+  public void clean_data_older_than_7_days() {
+    var snapshotDao = dbTester.getDbClient().snapshotDao();
+    var snapshot1 = createSnapshot(LocalDateTime.now().minusDays(1).toInstant(ZoneOffset.UTC).toEpochMilli());
+    snapshotDao.insert(dbSession, snapshot1);
+    scannerAnalysisCacheDao.insert(dbSession, snapshot1.getComponentUuid(), new ByteArrayInputStream("data".getBytes()));
+    var snapshot2 = createSnapshot(LocalDateTime.now().minusDays(6).toInstant(ZoneOffset.UTC).toEpochMilli());
+    snapshotDao.insert(dbSession, snapshot2);
+    scannerAnalysisCacheDao.insert(dbSession, snapshot2.getComponentUuid(), new ByteArrayInputStream("data".getBytes()));
+    var snapshot3 = createSnapshot(LocalDateTime.now().minusDays(8).toInstant(ZoneOffset.UTC).toEpochMilli());
+    snapshotDao.insert(dbSession, snapshot3);
+    scannerAnalysisCacheDao.insert(dbSession, snapshot3.getComponentUuid(), new ByteArrayInputStream("data".getBytes()));
+    var snapshot4 = createSnapshot(LocalDateTime.now().minusDays(30).toInstant(ZoneOffset.UTC).toEpochMilli());
+    snapshotDao.insert(dbSession, snapshot4);
+    scannerAnalysisCacheDao.insert(dbSession, snapshot4.getComponentUuid(), new ByteArrayInputStream("data".getBytes()));
+
+    assertThat(dbTester.countRowsOfTable("scanner_analysis_cache")).isEqualTo(4);
+
+    underTest.clean();
+
+    assertThat(dbTester.countRowsOfTable("scanner_analysis_cache")).isEqualTo(2);
+    assertThat(scannerAnalysisCacheDao.selectData(dbSession, snapshot1.getComponentUuid())).isNotNull();
+    assertThat(scannerAnalysisCacheDao.selectData(dbSession, snapshot2.getComponentUuid())).isNotNull();
+    assertThat(scannerAnalysisCacheDao.selectData(dbSession, snapshot3.getComponentUuid())).isNull();
+    assertThat(scannerAnalysisCacheDao.selectData(dbSession, snapshot4.getComponentUuid())).isNull();
+  }
+
+  private static SnapshotDto createSnapshot(long buildtime) {
+    return new SnapshotDto()
+      .setUuid(uuidFactory.create())
+      .setComponentUuid(uuidFactory.create())
+      .setStatus("P")
+      .setLast(true)
+      .setProjectVersion("2.1-SNAPSHOT")
+      .setPeriodMode("days1")
+      .setPeriodParam("30")
+      .setPeriodDate(buildtime)
+      .setBuildDate(buildtime);
+  }
+
+}