--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.ce.analysis.cache.cleaning;
+
+import java.util.concurrent.ScheduledExecutorService;
+
+public interface AnalysisCacheCleaningExecutorService extends ScheduledExecutorService {
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.ce.analysis.cache.cleaning;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import org.sonar.server.util.AbstractStoppableScheduledExecutorServiceImpl;
+
+public class AnalysisCacheCleaningExecutorServiceImpl extends AbstractStoppableScheduledExecutorServiceImpl<ScheduledExecutorService>
+ implements AnalysisCacheCleaningExecutorService {
+
+ public AnalysisCacheCleaningExecutorServiceImpl() {
+ super(Executors.newSingleThreadScheduledExecutor(
+ new ThreadFactoryBuilder()
+ .setDaemon(true)
+ .setNameFormat("Analysis_cache_cleaning-%d")
+ .build()));
+ }
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.ce.analysis.cache.cleaning;
+
+import org.sonar.core.platform.Module;
+
+public class AnalysisCacheCleaningModule extends Module {
+ @Override protected void configureModule() {
+ add(
+ AnalysisCacheCleaningExecutorServiceImpl.class,
+ AnalysisCacheCleaningSchedulerImpl.class
+ );
+ }
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.ce.analysis.cache.cleaning;
+
+import org.sonar.api.platform.ServerStartHandler;
+
+public interface AnalysisCacheCleaningScheduler extends ServerStartHandler {
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.ce.analysis.cache.cleaning;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.time.Duration;
+import java.time.LocalDateTime;
+import org.sonar.api.platform.Server;
+import org.sonar.db.DbClient;
+
+import static java.util.concurrent.TimeUnit.DAYS;
+import static java.util.concurrent.TimeUnit.SECONDS;
+
+public class AnalysisCacheCleaningSchedulerImpl implements AnalysisCacheCleaningScheduler {
+ private final AnalysisCacheCleaningExecutorService executorService;
+ private final DbClient dbClient;
+
+ public AnalysisCacheCleaningSchedulerImpl(AnalysisCacheCleaningExecutorService executorService, DbClient dbClient) {
+ this.executorService = executorService;
+ this.dbClient = dbClient;
+ }
+
+ @Override public void onServerStart(Server server) {
+ LocalDateTime now = LocalDateTime.now();
+ // schedule run at midnight everyday
+ LocalDateTime nextRun = now.plusDays(1).withHour(0).withMinute(0).withSecond(0);
+ long initialDelay = Duration.between(now, nextRun).getSeconds();
+ executorService.scheduleAtFixedRate(this::clean, initialDelay, DAYS.toSeconds(1), SECONDS);
+ }
+
+ @VisibleForTesting
+ void clean() {
+ try (var dbSession = dbClient.openSession(false)) {
+ dbClient.scannerAnalysisCacheDao().cleanOlderThan7Days(dbSession);
+ dbSession.commit();
+ }
+ }
+}
import org.sonar.ce.CeQueueModule;
import org.sonar.ce.CeTaskCommonsModule;
import org.sonar.ce.StandaloneCeDistributedInformation;
+import org.sonar.ce.analysis.cache.cleaning.AnalysisCacheCleaningModule;
import org.sonar.ce.async.SynchronousAsyncExecution;
import org.sonar.ce.cleaning.CeCleaningModule;
import org.sonar.ce.db.ReadOnlyPropertiesDao;
new WebhookModule(),
QualityGateFinder.class,
- QualityGateEvaluatorImpl.class
+ QualityGateEvaluatorImpl.class,
+
+ new AnalysisCacheCleaningModule()
);
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.ce.analysis.cache.cleaning;
+
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class AnalysisCacheCleaningExecutorServiceImplTest {
+
+ @Test
+ public void constructor_createsExecutorDelegateThatIsReadyToAct() {
+ AnalysisCacheCleaningExecutorServiceImpl underTest = new AnalysisCacheCleaningExecutorServiceImpl();
+
+ assertThat(underTest.isShutdown()).isFalse();
+ assertThat(underTest.isTerminated()).isFalse();
+ }
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.ce.analysis.cache.cleaning;
+
+import org.junit.Test;
+import org.sonar.core.platform.ListContainer;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class AnalysisCacheCleaningModuleTest {
+ @Test
+ public void verify_count_of_added_components() {
+ ListContainer container = new ListContainer();
+ new AnalysisCacheCleaningModule().configure(container);
+ assertThat(container.getAddedObjects()).hasSize(2);
+ }
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.ce.analysis.cache.cleaning;
+
+import java.io.ByteArrayInputStream;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import org.junit.Rule;
+import org.junit.Test;
+import org.sonar.api.platform.Server;
+import org.sonar.api.utils.System2;
+import org.sonar.core.util.SequenceUuidFactory;
+import org.sonar.core.util.UuidFactory;
+import org.sonar.db.DbSession;
+import org.sonar.db.DbTester;
+import org.sonar.db.component.SnapshotDto;
+import org.sonar.db.scannercache.ScannerAnalysisCacheDao;
+
+import static java.util.concurrent.TimeUnit.DAYS;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class AnalysisCacheCleaningSchedulerImplTest {
+ private System2 system2 = mock(System2.class);
+ private final static UuidFactory uuidFactory = new SequenceUuidFactory();
+ @Rule
+ public DbTester dbTester = DbTester.create(system2);
+ private DbSession dbSession = dbTester.getSession();
+ private ScannerAnalysisCacheDao scannerAnalysisCacheDao = dbTester.getDbClient().scannerAnalysisCacheDao();
+
+ AnalysisCacheCleaningExecutorService executorService = mock(AnalysisCacheCleaningExecutorService.class);
+
+ AnalysisCacheCleaningSchedulerImpl underTest = new AnalysisCacheCleaningSchedulerImpl(executorService, dbTester.getDbClient());
+
+ @Test
+ public void startSchedulingOnServerStart() {
+ underTest.onServerStart(mock(Server.class));
+ verify(executorService, times(1)).scheduleAtFixedRate(any(Runnable.class), anyLong(), eq(DAYS.toSeconds(1)), eq(SECONDS));
+ }
+
+ @Test
+ public void clean_data_older_than_7_days() {
+ var snapshotDao = dbTester.getDbClient().snapshotDao();
+ var snapshot1 = createSnapshot(LocalDateTime.now().minusDays(1).toInstant(ZoneOffset.UTC).toEpochMilli());
+ snapshotDao.insert(dbSession, snapshot1);
+ scannerAnalysisCacheDao.insert(dbSession, snapshot1.getComponentUuid(), new ByteArrayInputStream("data".getBytes()));
+ var snapshot2 = createSnapshot(LocalDateTime.now().minusDays(6).toInstant(ZoneOffset.UTC).toEpochMilli());
+ snapshotDao.insert(dbSession, snapshot2);
+ scannerAnalysisCacheDao.insert(dbSession, snapshot2.getComponentUuid(), new ByteArrayInputStream("data".getBytes()));
+ var snapshot3 = createSnapshot(LocalDateTime.now().minusDays(8).toInstant(ZoneOffset.UTC).toEpochMilli());
+ snapshotDao.insert(dbSession, snapshot3);
+ scannerAnalysisCacheDao.insert(dbSession, snapshot3.getComponentUuid(), new ByteArrayInputStream("data".getBytes()));
+ var snapshot4 = createSnapshot(LocalDateTime.now().minusDays(30).toInstant(ZoneOffset.UTC).toEpochMilli());
+ snapshotDao.insert(dbSession, snapshot4);
+ scannerAnalysisCacheDao.insert(dbSession, snapshot4.getComponentUuid(), new ByteArrayInputStream("data".getBytes()));
+
+ assertThat(dbTester.countRowsOfTable("scanner_analysis_cache")).isEqualTo(4);
+
+ underTest.clean();
+
+ assertThat(dbTester.countRowsOfTable("scanner_analysis_cache")).isEqualTo(2);
+ assertThat(scannerAnalysisCacheDao.selectData(dbSession, snapshot1.getComponentUuid())).isNotNull();
+ assertThat(scannerAnalysisCacheDao.selectData(dbSession, snapshot2.getComponentUuid())).isNotNull();
+ assertThat(scannerAnalysisCacheDao.selectData(dbSession, snapshot3.getComponentUuid())).isNull();
+ assertThat(scannerAnalysisCacheDao.selectData(dbSession, snapshot4.getComponentUuid())).isNull();
+ }
+
+ private static SnapshotDto createSnapshot(long buildtime) {
+ return new SnapshotDto()
+ .setUuid(uuidFactory.create())
+ .setComponentUuid(uuidFactory.create())
+ .setStatus("P")
+ .setLast(true)
+ .setProjectVersion("2.1-SNAPSHOT")
+ .setPeriodMode("days1")
+ .setPeriodParam("30")
+ .setPeriodDate(buildtime)
+ .setBuildDate(buildtime);
+ }
+
+}
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
import javax.annotation.CheckForNull;
import org.sonar.db.Dao;
import org.sonar.db.DbInputStream;
}
}
+ public void cleanOlderThan7Days(DbSession session) {
+ long timestamp = LocalDateTime.now().minusDays(7).toInstant(ZoneOffset.UTC).toEpochMilli();
+ mapper(session).cleanOlderThan(timestamp);
+ }
+
@CheckForNull
public DbInputStream selectData(DbSession dbSession, String branchUuid) {
PreparedStatement stmt = null;
void removeAll();
void remove(@Param("branchUuid") String branchUuid);
+
+ void cleanOlderThan(@Param("timestamp") long timestamp);
}
delete from scanner_analysis_cache where branch_uuid = #{branchUuid,jdbcType=VARCHAR}
</delete>
+ <delete id="cleanOlderThan">
+ delete from scanner_analysis_cache
+ where branch_uuid in (
+ select sac.branch_uuid from scanner_analysis_cache sac
+ left outer join snapshots s on
+ sac.branch_uuid = s.component_uuid
+ where
+ s.build_date < #{timestamp,jdbcType=BIGINT} and s.islast=${_true}
+ or s.islast is null
+ )
+ </delete>
+
</mapper>
import java.nio.charset.StandardCharsets;
import java.sql.Connection;
import java.sql.SQLException;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
import org.apache.commons.io.IOUtils;
import org.junit.Rule;
import org.junit.Test;
import org.sonar.api.utils.System2;
+import org.sonar.core.util.SequenceUuidFactory;
+import org.sonar.core.util.UuidFactory;
import org.sonar.db.DbInputStream;
import org.sonar.db.DbSession;
import org.sonar.db.DbTester;
+import org.sonar.db.component.SnapshotDto;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
public class ScannerAnalysisCacheDaoTest {
@Rule
public DbTester dbTester = DbTester.create(System2.INSTANCE);
-
+ private final static UuidFactory uuidFactory = new SequenceUuidFactory();
private final DbSession dbSession = dbTester.getSession();
private final ScannerAnalysisCacheDao underTest = dbTester.getDbClient().scannerAnalysisCacheDao();
.hasMessage("Fail to insert cache for branch uuid");
}
+ @Test
+ public void cleanOlderThan7Days() {
+ var snapshotDao = dbTester.getDbClient().snapshotDao();
+ var snapshot1 = createSnapshot(LocalDateTime.now().minusDays(1).toInstant(ZoneOffset.UTC).toEpochMilli());
+ snapshotDao.insert(dbSession, snapshot1);
+ underTest.insert(dbSession, snapshot1.getComponentUuid(), stringToInputStream("test data"));
+ var snapshot2 = createSnapshot(LocalDateTime.now().minusDays(6).toInstant(ZoneOffset.UTC).toEpochMilli());
+ snapshotDao.insert(dbSession, snapshot2);
+ underTest.insert(dbSession, snapshot2.getComponentUuid(), stringToInputStream("test data"));
+ var snapshot3 = createSnapshot(LocalDateTime.now().minusDays(8).toInstant(ZoneOffset.UTC).toEpochMilli());
+ snapshotDao.insert(dbSession, snapshot3);
+ underTest.insert(dbSession, snapshot3.getComponentUuid(), stringToInputStream("test data"));
+ var snapshot4 = createSnapshot(LocalDateTime.now().minusDays(30).toInstant(ZoneOffset.UTC).toEpochMilli());
+ snapshotDao.insert(dbSession, snapshot4);
+ underTest.insert(dbSession, snapshot4.getComponentUuid(), stringToInputStream("test data"));
+
+ assertThat(dbTester.countRowsOfTable("scanner_analysis_cache")).isEqualTo(4);
+
+ underTest.cleanOlderThan7Days(dbSession);
+ dbSession.commit();
+
+ assertThat(dbTester.countRowsOfTable("scanner_analysis_cache")).isEqualTo(2);
+ assertThat(underTest.selectData(dbSession, snapshot1.getComponentUuid())).isNotNull();
+ assertThat(underTest.selectData(dbSession, snapshot2.getComponentUuid())).isNotNull();
+ assertThat(underTest.selectData(dbSession, snapshot3.getComponentUuid())).isNull();
+ assertThat(underTest.selectData(dbSession, snapshot4.getComponentUuid())).isNull();
+ }
+
private static String dataStreamToString(DbInputStream dbInputStream) throws IOException {
try (DbInputStream is = dbInputStream) {
return IOUtils.toString(is, StandardCharsets.UTF_8);
return new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8));
}
+ private static SnapshotDto createSnapshot(long buildtime) {
+ return new SnapshotDto()
+ .setUuid(uuidFactory.create())
+ .setComponentUuid(uuidFactory.create())
+ .setStatus("P")
+ .setLast(true)
+ .setProjectVersion("2.1-SNAPSHOT")
+ .setPeriodMode("days1")
+ .setPeriodParam("30")
+ .setPeriodDate(buildtime)
+ .setBuildDate(buildtime);
+ }
+
}