From b7dec8c5218ba79517376612e47ab0c63aed466f Mon Sep 17 00:00:00 2001 From: belen-pruvost-sonarsource Date: Thu, 22 Jul 2021 21:01:19 +0200 Subject: [PATCH] SONAR-15144 - Audit Log Clean Up Job --- .../AuditHousekeepingFrequencyHelper.java | 66 +++++++++++ .../taskprocessor/AuditPurgeStep.java | 63 +++++++++++ .../taskprocessor/AuditPurgeTaskModule.java | 30 +++++ .../AuditPurgeTaskProcessor.java | 84 ++++++++++++++ .../AuditHousekeepingFrequencyHelperTest.java | 104 ++++++++++++++++++ .../taskprocessor/AuditPurgeStepTest.java | 99 +++++++++++++++++ .../AuditPurgeTaskModuleTest.java | 37 +++++++ .../AuditPurgeTaskProcessorTest.java | 72 ++++++++++++ .../container/ComputeEngineContainerImpl.java | 2 + .../java/org/sonar/db/audit/AuditDao.java | 10 +- .../java/org/sonar/db/audit/AuditMapper.java | 9 +- .../java/org/sonar/db/ce/CeTaskTypes.java | 3 +- .../org/sonar/db/audit/AuditMapper.xml | 18 ++- .../java/org/sonar/db/audit/AuditDaoTest.java | 9 +- .../platformlevel/PlatformLevel4.java | 2 + .../java/org/sonar/core/config/Frequency.java | 4 + .../resources/org/sonar/l10n/core.properties | 1 + 17 files changed, 602 insertions(+), 11 deletions(-) create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditHousekeepingFrequencyHelper.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditPurgeStep.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditPurgeTaskModule.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditPurgeTaskProcessor.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditHousekeepingFrequencyHelperTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditPurgeStepTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditPurgeTaskModuleTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditPurgeTaskProcessorTest.java diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditHousekeepingFrequencyHelper.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditHousekeepingFrequencyHelper.java new file mode 100644 index 00000000000..b35b84a29a8 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditHousekeepingFrequencyHelper.java @@ -0,0 +1,66 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.taskprocessor; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Arrays; +import java.util.Optional; +import org.sonar.api.utils.System2; +import org.sonar.core.config.Frequency; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.property.PropertyDto; + +import static org.sonar.core.config.PurgeConstants.AUDIT_HOUSEKEEPING_FREQUENCY; +import static org.sonar.core.config.PurgeProperties.DEFAULT_FREQUENCY; + +public class AuditHousekeepingFrequencyHelper { + private final System2 system2; + + public AuditHousekeepingFrequencyHelper(System2 system2) { + this.system2 = system2; + } + + public PropertyDto getHouseKeepingFrequency(DbClient dbClient, DbSession dbSession) { + return Optional.ofNullable(dbClient.propertiesDao() + .selectGlobalProperty(dbSession, AUDIT_HOUSEKEEPING_FREQUENCY)) + .orElse(defaultAuditHouseKeepingProperty()); + } + + public long getThresholdDate(String frequency) { + Optional housekeepingFrequency = Arrays.stream(Frequency.values()) + .filter(f -> f.name().equalsIgnoreCase(frequency)).findFirst(); + if (housekeepingFrequency.isEmpty()) { + throw new IllegalArgumentException("Unsupported frequency: " + frequency); + } + + return Instant.ofEpochMilli(system2.now()) + .minus(housekeepingFrequency.get().getDays(), ChronoUnit.DAYS) + .toEpochMilli(); + } + + private static PropertyDto defaultAuditHouseKeepingProperty() { + PropertyDto property = new PropertyDto(); + property.setKey(AUDIT_HOUSEKEEPING_FREQUENCY); + property.setValue(DEFAULT_FREQUENCY); + return property; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditPurgeStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditPurgeStep.java new file mode 100644 index 00000000000..af44ef8937f --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditPurgeStep.java @@ -0,0 +1,63 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.taskprocessor; + +import java.util.Set; +import java.util.stream.Collectors; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.audit.AuditDto; +import org.sonar.db.property.PropertyDto; + +public final class AuditPurgeStep implements ComputationStep { + private static final Logger LOG = Loggers.get(AuditPurgeStep.class); + + private final AuditHousekeepingFrequencyHelper auditHousekeepingFrequencyHelper; + private final DbClient dbClient; + + public AuditPurgeStep(AuditHousekeepingFrequencyHelper auditHousekeepingFrequencyHelper, DbClient dbClient) { + this.auditHousekeepingFrequencyHelper = auditHousekeepingFrequencyHelper; + this.dbClient = dbClient; + } + + @Override + public void execute(Context context) { + try (DbSession dbSession = dbClient.openSession(false)) { + PropertyDto property = auditHousekeepingFrequencyHelper.getHouseKeepingFrequency(dbClient, dbSession); + long deleteBefore = auditHousekeepingFrequencyHelper.getThresholdDate(property.getValue()); + Set auditUuids = dbClient.auditDao() + .selectOlderThan(dbSession, deleteBefore) + .stream() + .map(AuditDto::getUuid) + .collect(Collectors.toSet()); + LOG.info(String.format("%s audit logs to be deleted...", auditUuids.size())); + dbClient.auditDao().deleteByUuids(dbSession, auditUuids); + dbSession.commit(); + } + } + + @Override + public String getDescription() { + return "Purge Audit Logs"; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditPurgeTaskModule.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditPurgeTaskModule.java new file mode 100644 index 00000000000..936f9ed9f2d --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditPurgeTaskModule.java @@ -0,0 +1,30 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.taskprocessor; + +import org.sonar.core.platform.Module; + +public class AuditPurgeTaskModule extends Module { + + @Override + protected void configureModule() { + add(AuditPurgeTaskProcessor.class); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditPurgeTaskProcessor.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditPurgeTaskProcessor.java new file mode 100644 index 00000000000..14c1b034b94 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditPurgeTaskProcessor.java @@ -0,0 +1,84 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.taskprocessor; + +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import javax.annotation.CheckForNull; +import org.sonar.ce.task.CeTask; +import org.sonar.ce.task.CeTaskResult; +import org.sonar.ce.task.container.TaskContainer; +import org.sonar.ce.task.container.TaskContainerImpl; +import org.sonar.ce.task.projectanalysis.step.AbstractComputationSteps; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.ce.task.step.ComputationStepExecutor; +import org.sonar.ce.task.taskprocessor.CeTaskProcessor; +import org.sonar.core.platform.ComponentContainer; +import org.sonar.core.platform.ContainerPopulator; + +import static org.sonar.db.ce.CeTaskTypes.AUDIT_PURGE; + +public class AuditPurgeTaskProcessor implements CeTaskProcessor { + private static final Set HANDLED_TYPES = Set.of(AUDIT_PURGE); + + private final ComponentContainer ceEngineContainer; + + public AuditPurgeTaskProcessor(ComponentContainer ceEngineContainer) { + this.ceEngineContainer = ceEngineContainer; + } + + @Override + public Set getHandledCeTaskTypes() { + return HANDLED_TYPES; + } + + @CheckForNull + @Override + public CeTaskResult process(CeTask task) { + try (TaskContainer container = new TaskContainerImpl(ceEngineContainer, newContainerPopulator(task))) { + container.bootup(); + container.getComponentByType(ComputationStepExecutor.class).execute(); + } + return null; + } + + static ContainerPopulator newContainerPopulator(CeTask task) { + return taskContainer -> { + taskContainer.add(task); + taskContainer.add(AuditHousekeepingFrequencyHelper.class); + taskContainer.add(AuditPurgeStep.class); + taskContainer.add(new AuditPurgeComputationSteps(taskContainer)); + taskContainer.add(ComputationStepExecutor.class); + }; + } + + public static final class AuditPurgeComputationSteps extends AbstractComputationSteps { + + public AuditPurgeComputationSteps(ContainerPopulator.Container container) { + super(container); + } + + @Override + public List> orderedStepClasses() { + return Arrays.asList(AuditPurgeStep.class); + } + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditHousekeepingFrequencyHelperTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditHousekeepingFrequencyHelperTest.java new file mode 100644 index 00000000000..ce5a3df70dc --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditHousekeepingFrequencyHelperTest.java @@ -0,0 +1,104 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.taskprocessor; + +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.sonar.api.impl.utils.TestSystem2; +import org.sonar.api.utils.System2; +import org.sonar.core.config.Frequency; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.property.PropertiesDao; +import org.sonar.db.property.PropertyDto; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.sonar.core.config.PurgeConstants.AUDIT_HOUSEKEEPING_FREQUENCY; +import static org.sonar.core.config.PurgeProperties.DEFAULT_FREQUENCY; + +@RunWith(DataProviderRunner.class) +public class AuditHousekeepingFrequencyHelperTest { + private static final long NOW = 10_000_000_000L; + + private final DbClient dbClient = mock(DbClient.class); + private final DbSession dbSession = mock(DbSession.class); + private final PropertiesDao propertiesDao = mock(PropertiesDao.class); + private final System2 system2 = new TestSystem2().setNow(NOW); + private final AuditHousekeepingFrequencyHelper underTest = new AuditHousekeepingFrequencyHelper(system2); + + @Test + @UseDataProvider("frequencyOptions") + public void getThresholdDate(Frequency frequency) { + long result = underTest.getThresholdDate(frequency.getDescription()); + + + long expected = Instant.ofEpochMilli(system2.now()) + .minus(frequency.getDays(), ChronoUnit.DAYS) + .toEpochMilli(); + + assertThat(result).isEqualTo(expected); + } + + @Test + public void getThresholdDateForUnknownFrequencyFails() { + assertThatThrownBy(() -> underTest.getThresholdDate("Lalala")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unsupported frequency: Lalala"); + } + + @Test + public void getHouseKeepingFrequency() { + String value = "Weekly"; + PropertyDto propertyDto = new PropertyDto().setKey(AUDIT_HOUSEKEEPING_FREQUENCY).setValue(value); + when(dbClient.propertiesDao()).thenReturn(propertiesDao); + when(propertiesDao + .selectGlobalProperty(dbSession, AUDIT_HOUSEKEEPING_FREQUENCY)) + .thenReturn(propertyDto); + assertThat(underTest.getHouseKeepingFrequency(dbClient, dbSession).getValue()).isEqualTo(value); + } + + @Test + public void getDefaultHouseKeepingFrequencyWhenNotSet() { + when(dbClient.propertiesDao()).thenReturn(propertiesDao); + when(propertiesDao + .selectGlobalProperty(dbSession, AUDIT_HOUSEKEEPING_FREQUENCY)) + .thenReturn(null); + assertThat(underTest.getHouseKeepingFrequency(dbClient, dbSession).getValue()) + .isEqualTo(DEFAULT_FREQUENCY); + } + + @DataProvider + public static Object[][] frequencyOptions() { + return new Object[][] { + {Frequency.WEEKLY}, + {Frequency.MONTHLY}, + {Frequency.TRIMESTRIAL}, + {Frequency.YEARLY} + }; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditPurgeStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditPurgeStepTest.java new file mode 100644 index 00000000000..96d6e03d52f --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditPurgeStepTest.java @@ -0,0 +1,99 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.taskprocessor; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.api.utils.System2; +import org.sonar.db.DbClient; +import org.sonar.db.DbTester; +import org.sonar.db.audit.AuditDto; +import org.sonar.db.audit.AuditTesting; +import org.sonar.db.property.PropertyDto; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.sonar.core.config.Frequency.MONTHLY; +import static org.sonar.core.config.PurgeConstants.AUDIT_HOUSEKEEPING_FREQUENCY; + +public class AuditPurgeStepTest { + private final static long NOW = 1_400_000_000_000L; + private final static long BEFORE = 1_300_000_000_000L; + private final static long LATER = 1_500_000_000_000L; + private final static ZonedDateTime thresholdDate = Instant.ofEpochMilli(NOW) + .atZone(ZoneId.systemDefault()); + private final static PropertyDto FREQUENCY_PROPERTY = new PropertyDto() + .setKey(AUDIT_HOUSEKEEPING_FREQUENCY) + .setValue(MONTHLY.name()); + + @Rule + public final DbTester dbTester = DbTester.create(System2.INSTANCE); + + private final DbClient dbClient = dbTester.getDbClient(); + + private final System2 system2 = new System2(); + + @Rule + public final DbTester db = DbTester.create(system2); + + private final AuditHousekeepingFrequencyHelper auditHousekeepingFrequencyHelper = mock(AuditHousekeepingFrequencyHelper.class); + + private final AuditPurgeStep underTest = new AuditPurgeStep(auditHousekeepingFrequencyHelper, dbClient); + + @Before + public void setUp() { + when(auditHousekeepingFrequencyHelper.getHouseKeepingFrequency(any(), any())).thenReturn(FREQUENCY_PROPERTY); + when(auditHousekeepingFrequencyHelper.getThresholdDate(anyString())).thenReturn(NOW); + } + + @Test + public void executeDeletesOlderAudits() { + prepareRowsWithDeterministicCreatedAt(); + assertThat(dbClient.auditDao().selectOlderThan(db.getSession(), LATER + 1)).hasSize(3); + + underTest.execute(() -> null); + + assertThat(dbClient.auditDao().selectOlderThan(db.getSession(), LATER + 1)).hasSize(2); + } + + @Test + public void getDescription() { + assertThat(underTest.getDescription()).isEqualTo("Purge Audit Logs"); + } + + private void prepareRowsWithDeterministicCreatedAt() { + insertAudit(BEFORE); + insertAudit(NOW); + insertAudit(LATER); + db.getSession().commit(); + } + + private void insertAudit(long timestamp) { + AuditDto auditDto = AuditTesting.newAuditDto(timestamp); + dbClient.auditDao().insert(db.getSession(), auditDto); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditPurgeTaskModuleTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditPurgeTaskModuleTest.java new file mode 100644 index 00000000000..53857b49fa9 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditPurgeTaskModuleTest.java @@ -0,0 +1,37 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.taskprocessor; + +import org.junit.Test; +import org.sonar.core.platform.ComponentContainer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.sonar.core.platform.ComponentContainer.COMPONENTS_IN_EMPTY_COMPONENT_CONTAINER; + +public class AuditPurgeTaskModuleTest { + + @Test + public void verifyCountOfAddedComponents() { + ComponentContainer container = new ComponentContainer(); + new AuditPurgeTaskModule().configure(container); + assertThat(container.size()).isEqualTo(COMPONENTS_IN_EMPTY_COMPONENT_CONTAINER + 1); + } + +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditPurgeTaskProcessorTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditPurgeTaskProcessorTest.java new file mode 100644 index 00000000000..9e68167b02b --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditPurgeTaskProcessorTest.java @@ -0,0 +1,72 @@ +/* + * SonarQube + * Copyright (C) 2009-2021 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.taskprocessor; + +import java.util.List; +import org.assertj.core.api.Assertions; +import org.junit.Test; +import org.mockito.Mockito; +import org.sonar.ce.task.CeTask; +import org.sonar.ce.task.container.TaskContainer; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.core.platform.ComponentContainer; + +import static org.mockito.ArgumentMatchers.any; +import static org.sonar.ce.task.projectanalysis.taskprocessor.AuditPurgeTaskProcessor.AuditPurgeComputationSteps; +import static org.sonar.db.ce.CeTaskTypes.AUDIT_PURGE; + +public class AuditPurgeTaskProcessorTest { + + private ComponentContainer ceEngineContainer = Mockito.mock(ComponentContainer.class); + + private AuditPurgeTaskProcessor underTest = new AuditPurgeTaskProcessor(ceEngineContainer); + private TaskContainer container = Mockito.spy(TaskContainer.class); + + @Test + public void getHandledCeTaskTypes() { + Assertions.assertThat(underTest.getHandledCeTaskTypes()).containsExactly(AUDIT_PURGE); + } + + @Test(expected = NullPointerException.class) + public void processThrowsNPEIfCeTaskIsNull() { + underTest.process(null); + } + + @Test + public void newContainerPopulator() { + CeTask task = new CeTask.Builder() + .setUuid("TASK_UUID") + .setType("Type") + .build(); + + AuditPurgeTaskProcessor.newContainerPopulator(task).populateContainer(container); + Mockito.verify(container, Mockito.times(5)).add(any()); + } + + @Test + public void orderedStepClasses(){ + AuditPurgeComputationSteps auditPurgeComputationSteps = new AuditPurgeComputationSteps(null); + + List> steps = auditPurgeComputationSteps.orderedStepClasses(); + + Assertions.assertThat(steps).containsExactly(AuditPurgeStep.class); + } + +} diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java b/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java index ddb8a16dd86..e8204854e61 100644 --- a/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java +++ b/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java @@ -63,6 +63,7 @@ import org.sonar.ce.task.projectanalysis.ProjectAnalysisTaskModule; import org.sonar.ce.task.projectanalysis.analysis.ProjectConfigurationFactory; import org.sonar.ce.task.projectanalysis.issue.AdHocRuleCreator; import org.sonar.ce.task.projectanalysis.notification.ReportAnalysisFailureNotificationModule; +import org.sonar.ce.task.projectanalysis.taskprocessor.AuditPurgeTaskModule; import org.sonar.ce.task.projectanalysis.taskprocessor.IssueSyncTaskModule; import org.sonar.ce.taskprocessor.CeProcessingScheduler; import org.sonar.ce.taskprocessor.CeTaskProcessorModule; @@ -433,6 +434,7 @@ public class ComputeEngineContainerImpl implements ComputeEngineContainer { CeTaskCommonsModule.class, ProjectAnalysisTaskModule.class, IssueSyncTaskModule.class, + AuditPurgeTaskModule.class, CeTaskProcessorModule.class, OfficialDistribution.class, diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/audit/AuditDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/audit/AuditDao.java index 7c2b2d67979..75a8ed49b7b 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/audit/AuditDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/audit/AuditDao.java @@ -20,12 +20,14 @@ package org.sonar.db.audit; import java.util.List; +import java.util.Set; import org.sonar.api.utils.System2; import org.sonar.core.util.UuidFactory; import org.sonar.db.Dao; import org.sonar.db.DbSession; import org.sonar.db.Pagination; +import static org.sonar.db.DatabaseUtils.executeLargeUpdates; import static org.sonar.server.platform.db.migration.def.VarcharColumnDef.MAX_SIZE; public class AuditDao implements Dao { @@ -63,7 +65,11 @@ public class AuditDao implements Dao { getMapper(dbSession).insert(auditDto); } - public void deleteIfBeforeSelectedDate(DbSession dbSession, long timestamp) { - getMapper(dbSession).deleteIfBeforeSelectedDate(timestamp); + public List selectOlderThan(DbSession dbSession, long beforeTimestamp) { + return getMapper(dbSession).selectOlderThan(beforeTimestamp); + } + + public void deleteByUuids(DbSession dbSession, Set uuids) { + executeLargeUpdates(uuids, getMapper(dbSession)::deleteByUuids); } } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/audit/AuditMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/audit/AuditMapper.java index f5a45d34ba6..f252f36bf19 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/audit/AuditMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/audit/AuditMapper.java @@ -19,19 +19,20 @@ */ package org.sonar.db.audit; +import java.util.List; import org.apache.ibatis.annotations.Param; import org.sonar.db.Pagination; -import java.util.List; - public interface AuditMapper { void insert(@Param("dto") AuditDto auditDto); void delete(@Param("uuids") List uuids); - void deleteIfBeforeSelectedDate(@Param("timestamp") long timestamp); - List selectByPeriodPaginated(@Param("start")long start, @Param("end") long end, @Param("pagination") Pagination pagination); + List selectOlderThan(@Param("beforeTimestamp") long beforeTimestamp); + + void deleteByUuids(@Param("uuids") List uuids); + } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskTypes.java b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskTypes.java index fc1aadcc181..aae74db3d32 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskTypes.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskTypes.java @@ -21,8 +21,9 @@ package org.sonar.db.ce; public final class CeTaskTypes { - public static final String REPORT = "REPORT"; + public static final String AUDIT_PURGE = "AUDIT_PURGE"; public static final String BRANCH_ISSUE_SYNC = "ISSUE_SYNC"; + public static final String REPORT = "REPORT"; private CeTaskTypes() { // only statics diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/audit/AuditMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/audit/AuditMapper.xml index a2a6347c333..469560c01a5 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/audit/AuditMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/audit/AuditMapper.xml @@ -47,10 +47,22 @@ ) - - delete from audits + + + + delete + from audits + where + uuid in + + #{uuid,jdbcType=VARCHAR} + diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/audit/AuditDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/audit/AuditDaoTest.java index b338b23132d..3634503a394 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/audit/AuditDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/audit/AuditDaoTest.java @@ -20,6 +20,8 @@ package org.sonar.db.audit; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import org.junit.Rule; import org.junit.Test; import org.sonar.api.impl.utils.TestSystem2; @@ -63,7 +65,12 @@ public class AuditDaoTest { public void deleteIfBeforeSelectedDate_deleteTwoRows() { prepareRowsWithDeterministicCreatedAt(3); - testAuditDao.deleteIfBeforeSelectedDate(dbSession, 2); + Set auditUuids = testAuditDao.selectOlderThan(dbSession, 3) + .stream() + .map(AuditDto::getUuid) + .collect(Collectors.toSet()); + + testAuditDao.deleteByUuids(dbSession, auditUuids); List auditDtos = testAuditDao.selectByPeriodPaginated(dbSession, 1, 4, 1); assertThat(auditDtos.size()).isEqualTo(1); diff --git a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java index 69c70726abc..318df07af2d 100644 --- a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java +++ b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java @@ -39,6 +39,7 @@ import org.sonar.auth.gitlab.GitLabModule; import org.sonar.auth.ldap.LdapModule; import org.sonar.auth.saml.SamlModule; import org.sonar.ce.task.projectanalysis.notification.ReportAnalysisFailureNotificationModule; +import org.sonar.ce.task.projectanalysis.taskprocessor.AuditPurgeTaskProcessor; import org.sonar.ce.task.projectanalysis.taskprocessor.IssueSyncTaskProcessor; import org.sonar.ce.task.projectanalysis.taskprocessor.ReportTaskProcessor; import org.sonar.core.component.DefaultResourceTypes; @@ -523,6 +524,7 @@ public class PlatformLevel4 extends PlatformLevel { CeWsModule.class, ReportTaskProcessor.class, IssueSyncTaskProcessor.class, + AuditPurgeTaskProcessor.class, // SonarSource editions PlatformEditionProvider.class, diff --git a/sonar-core/src/main/java/org/sonar/core/config/Frequency.java b/sonar-core/src/main/java/org/sonar/core/config/Frequency.java index 4fc9ee46c2a..869c9d98f68 100644 --- a/sonar-core/src/main/java/org/sonar/core/config/Frequency.java +++ b/sonar-core/src/main/java/org/sonar/core/config/Frequency.java @@ -32,6 +32,10 @@ public enum Frequency { return this.description; } + public int getDays() { + return this.days; + } + Frequency(String description, int days) { this.description = description; this.days = days; diff --git a/sonar-core/src/main/resources/org/sonar/l10n/core.properties b/sonar-core/src/main/resources/org/sonar/l10n/core.properties index 4b4bb3eaa3e..8c92018ee34 100644 --- a/sonar-core/src/main/resources/org/sonar/l10n/core.properties +++ b/sonar-core/src/main/resources/org/sonar/l10n/core.properties @@ -2862,6 +2862,7 @@ background_task.type.ISSUE_SYNC=Project Data Reload background_task.type.APP_REFRESH=Recomputation background_task.type.PROJECT_EXPORT=Project Export background_task.type.PROJECT_IMPORT=Project Import +background_task.type.AUDIT_PURGE=Audit Log Purge background_tasks.page=Background Tasks background_tasks.page.description=This page allows monitoring of the queue of tasks running asynchronously on the server. It also gives access to the history of finished tasks and their status. Analysis report processing is the most common kind of background task. -- 2.39.5