diff options
author | Duarte Meneses <duarte.meneses@sonarsource.com> | 2022-03-14 11:27:18 -0500 |
---|---|---|
committer | sonartech <sonartech@sonarsource.com> | 2022-03-18 20:02:57 +0000 |
commit | 1269984e8e09338c057d068d715ade7df5a0c354 (patch) | |
tree | b483c992fd8c57b79188c936fb9bfb6ff7955ed8 /server | |
parent | e987de516356fe37ee8eb35793d5fd709e2a099b (diff) | |
download | sonarqube-1269984e8e09338c057d068d715ade7df5a0c354.tar.gz sonarqube-1269984e8e09338c057d068d715ade7df5a0c354.zip |
SONAR-16115 Store plugin's scanner cache in SonarQube
Diffstat (limited to 'server')
26 files changed, 580 insertions, 42 deletions
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReader.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReader.java index 2bde5066b1f..2ef9211860c 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReader.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReader.java @@ -19,6 +19,7 @@ */ package org.sonar.ce.task.projectanalysis.batch; +import java.io.InputStream; import java.util.Optional; import javax.annotation.CheckForNull; import org.sonar.core.util.CloseableIterator; @@ -27,6 +28,9 @@ import org.sonar.scanner.protocol.output.ScannerReport; public interface BatchReportReader { ScannerReport.Metadata readMetadata(); + @CheckForNull + InputStream getPluginCache(); + CloseableIterator<String> readScannerLogs(); CloseableIterator<ScannerReport.ActiveRule> readActiveRules(); diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderImpl.java index 62c989e48a0..2e8018aaf57 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderImpl.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderImpl.java @@ -21,6 +21,7 @@ package org.sonar.ce.task.projectanalysis.batch; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.util.NoSuchElementException; import java.util.Optional; @@ -61,6 +62,13 @@ public class BatchReportReaderImpl implements BatchReportReader { return this.metadata; } + @CheckForNull + @Override + public InputStream getPluginCache() { + ensureInitialized(); + return delegate.getPluginCache(); + } + @Override public CloseableIterator<String> readScannerLogs() { ensureInitialized(); diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ExtractReportStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ExtractReportStep.java index 8da46b7cc41..13662b05974 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ExtractReportStep.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ExtractReportStep.java @@ -32,9 +32,9 @@ import org.sonar.api.utils.log.Loggers; import org.sonar.ce.task.CeTask; import org.sonar.ce.task.projectanalysis.batch.MutableBatchReportDirectoryHolder; import org.sonar.ce.task.step.ComputationStep; +import org.sonar.db.DbInputStream; import org.sonar.db.DbClient; import org.sonar.db.DbSession; -import org.sonar.db.ce.CeTaskInputDao; import org.sonar.process.FileUtils2; import static org.sonar.core.util.FileUtils.humanReadableByteCountSI; @@ -63,11 +63,11 @@ public class ExtractReportStep implements ComputationStep { @Override public void execute(ComputationStep.Context context) { try (DbSession dbSession = dbClient.openSession(false)) { - Optional<CeTaskInputDao.DataStream> opt = dbClient.ceTaskInputDao().selectData(dbSession, task.getUuid()); + Optional<DbInputStream> opt = dbClient.ceTaskInputDao().selectData(dbSession, task.getUuid()); if (opt.isPresent()) { File unzippedDir = tempFolder.newDir(); - try (CeTaskInputDao.DataStream reportStream = opt.get(); - InputStream zipStream = new BufferedInputStream(reportStream.getInputStream())) { + try (DbInputStream reportStream = opt.get(); + InputStream zipStream = new BufferedInputStream(reportStream)) { ZipUtils.unzip(zipStream, unzippedDir); } catch (IOException e) { throw new IllegalStateException("Fail to extract report " + task.getUuid() + " from database", e); diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistPluginCacheStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistPluginCacheStep.java new file mode 100644 index 00000000000..b135a6640fb --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistPluginCacheStep.java @@ -0,0 +1,64 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.step; + +import java.io.IOException; +import java.io.InputStream; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; +import org.sonar.ce.task.projectanalysis.batch.BatchReportReader; +import org.sonar.ce.task.projectanalysis.component.TreeRootHolder; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.db.DbClient; + +public class PersistPluginCacheStep implements ComputationStep { + private static final Logger LOGGER = Loggers.get(PersistPluginCacheStep.class); + private final BatchReportReader reportReader; + private final DbClient dbClient; + private final TreeRootHolder treeRootHolder; + + public PersistPluginCacheStep(BatchReportReader reportReader, DbClient dbClient, TreeRootHolder treeRootHolder) { + this.reportReader = reportReader; + this.dbClient = dbClient; + this.treeRootHolder = treeRootHolder; + } + + @Override + public String getDescription() { + return "Persist scanner plugin cache"; + } + + @Override + public void execute(ComputationStep.Context context) { + InputStream pluginCacheStream = reportReader.getPluginCache(); + if (pluginCacheStream != null) { + try (var dataStream = pluginCacheStream; + var dbSession = dbClient.openSession(false)) { + String branchUuid = treeRootHolder.getRoot().getUuid(); + dbClient.scannerCacheDao().remove(dbSession, branchUuid); + dbClient.scannerCacheDao().insert(dbSession, branchUuid, dataStream); + dbSession.commit(); + } catch (IOException e) { + LOGGER.error("Error in reading plugin cache", e); + } + } + + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java index 4df290ed19f..87d442e5d17 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java @@ -48,6 +48,7 @@ public class ReportComputationSteps extends AbstractComputationSteps { BuildComponentTreeStep.class, ValidateProjectStep.class, + PersistPluginCacheStep.class, LoadQualityProfilesStep.class, // load project related stuffs diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderRule.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderRule.java index b72e371ec72..45caf73d932 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderRule.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderRule.java @@ -20,6 +20,8 @@ package org.sonar.ce.task.projectanalysis.batch; import com.google.common.base.Preconditions; +import java.io.ByteArrayInputStream; +import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -57,6 +59,7 @@ public class BatchReportReaderRule implements TestRule, BatchReportReader { private Map<Integer, List<ScannerReport.LineSgnificantCode>> significantCode = new HashMap<>(); private Map<Integer, ScannerReport.ChangedLines> changedLines = new HashMap<>(); private List<ScannerReport.AnalysisWarning> analysisWarnings = Collections.emptyList(); + private byte[] pluginCache; @Override public Statement apply(final Statement statement, Description description) { @@ -106,6 +109,19 @@ public class BatchReportReaderRule implements TestRule, BatchReportReader { return metadata; } + @CheckForNull + @Override + public InputStream getPluginCache() { + if (pluginCache == null) { + return null; + } + return new ByteArrayInputStream(pluginCache); + } + + public void setPluginCache(byte[] cache) { + this.pluginCache = cache; + } + public BatchReportReaderRule setMetadata(ScannerReport.Metadata metadata) { this.metadata = metadata; return this; diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistPluginCacheStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistPluginCacheStepTest.java new file mode 100644 index 00000000000..c9f79e20edd --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistPluginCacheStepTest.java @@ -0,0 +1,69 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.step; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import org.apache.commons.io.IOUtils; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.ce.task.projectanalysis.batch.BatchReportReaderRule; +import org.sonar.ce.task.projectanalysis.component.Component; +import org.sonar.ce.task.projectanalysis.component.TreeRootHolderRule; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.db.DbClient; +import org.sonar.db.DbInputStream; +import org.sonar.db.DbTester; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class PersistPluginCacheStepTest { + @Rule + public BatchReportReaderRule reader = new BatchReportReaderRule(); + @Rule + public DbTester dbTester = DbTester.create(); + private final DbClient client = dbTester.getDbClient(); + private final TreeRootHolderRule treeRootHolder = new TreeRootHolderRule(); + private final PersistPluginCacheStep step = new PersistPluginCacheStep(reader, dbTester.getDbClient(), treeRootHolder); + + @Test + public void inserts_cache() throws IOException { + reader.setPluginCache("test".getBytes(UTF_8)); + + Component root = mock(Component.class); + when(root.getUuid()).thenReturn("branch"); + treeRootHolder.setRoot(root); + + step.execute(mock(ComputationStep.Context.class)); + assertThat(dbTester.countRowsOfTable("scanner_cache")).isOne(); + try (DbInputStream data = client.scannerCacheDao().selectData(dbTester.getSession(), "branch")) { + assertThat(IOUtils.toString(data, UTF_8)).isEqualTo("test"); + } + } + + @Test + public void updates_cache() throws IOException { + client.scannerCacheDao().insert(dbTester.getSession(), "branch", new ByteArrayInputStream("test".getBytes(UTF_8))); + inserts_cache(); + } +} diff --git a/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java b/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java index a7e50e7d4d1..413dc25ac6a 100644 --- a/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java +++ b/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java @@ -94,6 +94,7 @@ public final class SqTables { "rules_parameters", "rules_profiles", "rule_repositories", + "scanner_cache", "schema_migrations", "session_tokens", "snapshots", diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/DaoModule.java b/server/sonar-db-dao/src/main/java/org/sonar/db/DaoModule.java index b07fd319638..4649066af46 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/DaoModule.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/DaoModule.java @@ -77,6 +77,7 @@ import org.sonar.db.qualityprofile.QualityProfileDao; import org.sonar.db.qualityprofile.QualityProfileExportDao; import org.sonar.db.rule.RuleDao; import org.sonar.db.rule.RuleRepositoryDao; +import org.sonar.db.scannercache.ScannerCacheDao; import org.sonar.db.schemamigration.SchemaMigrationDao; import org.sonar.db.source.FileSourceDao; import org.sonar.db.user.GroupDao; @@ -156,6 +157,7 @@ public class DaoModule extends Module { RuleDao.class, RuleRepositoryDao.class, SamlMessageIdDao.class, + ScannerCacheDao.class, SnapshotDao.class, SchemaMigrationDao.class, SessionTokensDao.class, diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/DbClient.java b/server/sonar-db-dao/src/main/java/org/sonar/db/DbClient.java index 260f9430cdd..2268ea41b53 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/DbClient.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/DbClient.java @@ -57,8 +57,8 @@ import org.sonar.db.permission.template.PermissionTemplateCharacteristicDao; import org.sonar.db.permission.template.PermissionTemplateDao; import org.sonar.db.plugin.PluginDao; import org.sonar.db.portfolio.PortfolioDao; -import org.sonar.db.project.ProjectDao; import org.sonar.db.project.ProjectBadgeTokenDao; +import org.sonar.db.project.ProjectDao; import org.sonar.db.property.InternalComponentPropertiesDao; import org.sonar.db.property.InternalPropertiesDao; import org.sonar.db.property.PropertiesDao; @@ -77,6 +77,7 @@ import org.sonar.db.qualityprofile.QualityProfileDao; import org.sonar.db.qualityprofile.QualityProfileExportDao; import org.sonar.db.rule.RuleDao; import org.sonar.db.rule.RuleRepositoryDao; +import org.sonar.db.scannercache.ScannerCacheDao; import org.sonar.db.schemamigration.SchemaMigrationDao; import org.sonar.db.source.FileSourceDao; import org.sonar.db.user.GroupDao; @@ -168,6 +169,7 @@ public class DbClient { private final UserDismissedMessagesDao userDismissedMessagesDao; private final ApplicationProjectsDao applicationProjectsDao; private final ProjectBadgeTokenDao projectBadgeTokenDao; + private final ScannerCacheDao scannerCacheDao; public DbClient(Database database, MyBatis myBatis, DBSessions dbSessions, Dao... daos) { this.database = database; @@ -248,6 +250,7 @@ public class DbClient { samlMessageIdDao = getDao(map, SamlMessageIdDao.class); userDismissedMessagesDao = getDao(map, UserDismissedMessagesDao.class); applicationProjectsDao = getDao(map, ApplicationProjectsDao.class); + scannerCacheDao = getDao(map, ScannerCacheDao.class); } public DbSession openSession(boolean batch) { @@ -547,4 +550,8 @@ public class DbClient { public ProjectBadgeTokenDao projectBadgeTokenDao() { return projectBadgeTokenDao; } + + public ScannerCacheDao scannerCacheDao() { + return scannerCacheDao; + } } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/DbInputStream.java b/server/sonar-db-dao/src/main/java/org/sonar/db/DbInputStream.java new file mode 100644 index 00000000000..e956ad4e8b7 --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/DbInputStream.java @@ -0,0 +1,46 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db; + +import java.io.InputStream; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import org.apache.commons.io.IOUtils; +import org.sonar.api.internal.apachecommons.io.input.ProxyInputStream; + +public class DbInputStream extends ProxyInputStream { + private final PreparedStatement stmt; + private final ResultSet rs; + private final InputStream stream; + + public DbInputStream(PreparedStatement stmt, ResultSet rs, InputStream stream) { + super(stream); + this.stmt = stmt; + this.rs = rs; + this.stream = stream; + } + + @Override + public void close() { + IOUtils.closeQuietly(stream); + DatabaseUtils.closeQuietly(rs); + DatabaseUtils.closeQuietly(stmt); + } +} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java b/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java index 6917d3c656f..6792608657e 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java @@ -136,6 +136,7 @@ import org.sonar.db.rule.RuleDto; import org.sonar.db.rule.RuleMapper; import org.sonar.db.rule.RuleParamDto; import org.sonar.db.rule.RuleRepositoryMapper; +import org.sonar.db.scannercache.ScannerCacheMapper; import org.sonar.db.schemamigration.SchemaMigrationDto; import org.sonar.db.schemamigration.SchemaMigrationMapper; import org.sonar.db.source.FileSourceMapper; @@ -299,6 +300,7 @@ public class MyBatis { RuleMapper.class, RuleRepositoryMapper.class, SamlMessageIdMapper.class, + ScannerCacheMapper.class, SchemaMigrationMapper.class, SessionTokenMapper.class, SnapshotMapper.class, diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskInputDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskInputDao.java index a83dfb6801e..74e0b66b954 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskInputDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskInputDao.java @@ -27,9 +27,9 @@ import java.sql.SQLException; import java.util.Collection; import java.util.List; import java.util.Optional; -import org.apache.commons.io.IOUtils; import org.sonar.api.utils.System2; import org.sonar.db.Dao; +import org.sonar.db.DbInputStream; import org.sonar.db.DatabaseUtils; import org.sonar.db.DbSession; @@ -57,16 +57,16 @@ public class CeTaskInputDao implements Dao { } } - public Optional<DataStream> selectData(DbSession dbSession, String taskUuid) { + public Optional<DbInputStream> selectData(DbSession dbSession, String taskUuid) { PreparedStatement stmt = null; ResultSet rs = null; - DataStream result = null; + DbInputStream result = null; try { stmt = dbSession.getConnection().prepareStatement("SELECT input_data FROM ce_task_input WHERE task_uuid=? AND input_data IS NOT NULL"); stmt.setString(1, taskUuid); rs = stmt.executeQuery(); if (rs.next()) { - result = new DataStream(stmt, rs, rs.getBinaryStream(1)); + result = new DbInputStream(stmt, rs, rs.getBinaryStream(1)); return Optional.of(result); } return Optional.empty(); @@ -88,27 +88,4 @@ public class CeTaskInputDao implements Dao { CeTaskInputMapper mapper = dbSession.getMapper(CeTaskInputMapper.class); DatabaseUtils.executeLargeUpdates(uuids, mapper::deleteByUuids); } - - public static class DataStream implements AutoCloseable { - private final PreparedStatement stmt; - private final ResultSet rs; - private final InputStream stream; - - private DataStream(PreparedStatement stmt, ResultSet rs, InputStream stream) { - this.stmt = stmt; - this.rs = rs; - this.stream = stream; - } - - public InputStream getInputStream() { - return stream; - } - - @Override - public void close() { - IOUtils.closeQuietly(stream); - DatabaseUtils.closeQuietly(rs); - DatabaseUtils.closeQuietly(stmt); - } - } } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java index 80c818cfb27..c5d24de7b82 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java @@ -485,4 +485,11 @@ class PurgeCommands { profiler.stop(); } + public void deleteScannerCache(String rootUuid) { + profiler.start("deleteScannerCache (scanner_cache)"); + purgeMapper.deleteScannerCacheByBranchUuid(rootUuid); + session.commit(); + profiler.stop(); + } + } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java index db1ec59a32a..ddbe7b627df 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java @@ -175,4 +175,5 @@ public interface PurgeMapper { void deleteUserDismissedMessagesByProjectUuid(@Param("projectUuid") String projectUuid); + void deleteScannerCacheByBranchUuid(@Param("branchUuid") String branchUuid); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/scannercache/ScannerCacheDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/scannercache/ScannerCacheDao.java new file mode 100644 index 00000000000..717180948d9 --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/scannercache/ScannerCacheDao.java @@ -0,0 +1,82 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.scannercache; + +import java.io.InputStream; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import javax.annotation.CheckForNull; +import org.sonar.db.Dao; +import org.sonar.db.DbInputStream; +import org.sonar.db.DatabaseUtils; +import org.sonar.db.DbSession; + +public class ScannerCacheDao implements Dao { + public void removeAll(DbSession session) { + mapper(session).removeAll(); + } + + public void remove(DbSession session, String branchUuid) { + mapper(session).remove(branchUuid); + } + + public void insert(DbSession dbSession, String branchUuid, InputStream data) { + Connection connection = dbSession.getConnection(); + try (PreparedStatement stmt = connection.prepareStatement( + "INSERT INTO scanner_cache (branch_uuid, data) VALUES (?, ?)")) { + stmt.setString(1, branchUuid); + stmt.setBinaryStream(2, data); + stmt.executeUpdate(); + connection.commit(); + } catch (SQLException e) { + throw new IllegalStateException("Fail to insert cache for branch " + branchUuid, e); + } + } + + @CheckForNull + public DbInputStream selectData(DbSession dbSession, String branchUuid) { + PreparedStatement stmt = null; + ResultSet rs = null; + DbInputStream result = null; + try { + stmt = dbSession.getConnection().prepareStatement("SELECT data FROM scanner_cache WHERE branch_uuid=?"); + stmt.setString(1, branchUuid); + rs = stmt.executeQuery(); + if (rs.next()) { + result = new DbInputStream(stmt, rs, rs.getBinaryStream(1)); + return result; + } + return null; + } catch (SQLException e) { + throw new IllegalStateException("Fail to select cache for branch " + branchUuid, e); + } finally { + if (result == null) { + DatabaseUtils.closeQuietly(rs); + DatabaseUtils.closeQuietly(stmt); + } + } + } + + private static ScannerCacheMapper mapper(DbSession session) { + return session.getMapper(ScannerCacheMapper.class); + } +} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/scannercache/ScannerCacheMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/scannercache/ScannerCacheMapper.java new file mode 100644 index 00000000000..200d5eb1a38 --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/scannercache/ScannerCacheMapper.java @@ -0,0 +1,28 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.scannercache; + +import org.apache.ibatis.annotations.Param; + +public interface ScannerCacheMapper { + void removeAll(); + + void remove(@Param("branchUuid") String branchUuid); +} diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml index 8b21a47bfe7..738bf2e737d 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml @@ -592,5 +592,9 @@ <delete id="deleteUserDismissedMessagesByProjectUuid"> delete from user_dismissed_messages where project_uuid = #{projectUuid,jdbcType=VARCHAR} </delete> + + <delete id="deleteScannerCacheByBranchUuid"> + delete from scanner_cache where branch_uuid = #{branchUuid,jdbcType=VARCHAR} + </delete> </mapper> diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/scannercache/ScannerCacheMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/scannercache/ScannerCacheMapper.xml new file mode 100644 index 00000000000..887f694c8c0 --- /dev/null +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/scannercache/ScannerCacheMapper.xml @@ -0,0 +1,15 @@ +<?xml version="1.0" encoding="UTF-8" ?> + +<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "mybatis-3-mapper.dtd"> + +<mapper namespace="org.sonar.db.scannercache.ScannerCacheMapper"> + <delete id="removeAll"> + delete from scanner_cache + </delete> + + <delete id="remove"> + delete from scanner_cache where branch_uuid = #{branchUuid,jdbcType=VARCHAR} + </delete> + +</mapper> + diff --git a/server/sonar-db-dao/src/schema/schema-sq.ddl b/server/sonar-db-dao/src/schema/schema-sq.ddl index 7d0350a8626..1e5bdad6f2c 100644 --- a/server/sonar-db-dao/src/schema/schema-sq.ddl +++ b/server/sonar-db-dao/src/schema/schema-sq.ddl @@ -896,6 +896,12 @@ CREATE TABLE "SAML_MESSAGE_IDS"( ALTER TABLE "SAML_MESSAGE_IDS" ADD CONSTRAINT "PK_SAML_MESSAGE_IDS" PRIMARY KEY("UUID"); CREATE UNIQUE INDEX "SAML_MESSAGE_IDS_UNIQUE" ON "SAML_MESSAGE_IDS"("MESSAGE_ID" NULLS FIRST); +CREATE TABLE "SCANNER_CACHE"( + "BRANCH_UUID" CHARACTER VARYING(40) NOT NULL, + "DATA" BINARY LARGE OBJECT NOT NULL +); +ALTER TABLE "SCANNER_CACHE" ADD CONSTRAINT "PK_SCANNER_CACHE" PRIMARY KEY("BRANCH_UUID"); + CREATE TABLE "SESSION_TOKENS"( "UUID" CHARACTER VARYING(40) NOT NULL, "USER_UUID" CHARACTER VARYING(255) NOT NULL, diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeTaskInputDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeTaskInputDaoTest.java index c04246dff04..67d26127e97 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeTaskInputDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeTaskInputDaoTest.java @@ -25,8 +25,10 @@ import org.apache.commons.io.IOUtils; import org.junit.Rule; import org.junit.Test; import org.sonar.api.utils.System2; +import org.sonar.db.DbInputStream; import org.sonar.db.DbTester; +import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Collections.singleton; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -52,12 +54,10 @@ public class CeTaskInputDaoTest { InputStream report = IOUtils.toInputStream(SOME_DATA); underTest.insert(dbTester.getSession(), A_UUID, report); - Optional<CeTaskInputDao.DataStream> result = underTest.selectData(dbTester.getSession(), A_UUID); + Optional<DbInputStream> result = underTest.selectData(dbTester.getSession(), A_UUID); assertThat(result).isPresent(); - try { - assertThat(IOUtils.toString(result.get().getInputStream())).isEqualTo(SOME_DATA); - } finally { - result.get().close(); + try (DbInputStream is = result.get()) { + assertThat(IOUtils.toString(is, UTF_8)).isEqualTo(SOME_DATA); } } @@ -69,7 +69,7 @@ public class CeTaskInputDaoTest { @Test public void selectData_returns_absent_if_uuid_not_found() { - Optional<CeTaskInputDao.DataStream> result = underTest.selectData(dbTester.getSession(), A_UUID); + Optional<DbInputStream> result = underTest.selectData(dbTester.getSession(), A_UUID); assertThat(result).isNotPresent(); } @@ -78,7 +78,7 @@ public class CeTaskInputDaoTest { insertData(A_UUID); dbTester.commit(); - Optional<CeTaskInputDao.DataStream> result = underTest.selectData(dbTester.getSession(), A_UUID); + Optional<DbInputStream> result = underTest.selectData(dbTester.getSession(), A_UUID); assertThat(result).isNotPresent(); } diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java index e11b1991256..de70c5ebfc1 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java @@ -44,6 +44,7 @@ import org.sonar.api.utils.System2; import org.sonar.core.util.CloseableIterator; import org.sonar.core.util.UuidFactoryFast; import org.sonar.core.util.Uuids; +import org.sonar.db.DbInputStream; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.DbTester; @@ -52,7 +53,6 @@ import org.sonar.db.ce.CeActivityDto; import org.sonar.db.ce.CeQueueDto; import org.sonar.db.ce.CeQueueDto.Status; import org.sonar.db.ce.CeTaskCharacteristicDto; -import org.sonar.db.ce.CeTaskInputDao; import org.sonar.db.ce.CeTaskMessageDto; import org.sonar.db.ce.CeTaskMessageType; import org.sonar.db.ce.CeTaskTypes; @@ -1675,7 +1675,7 @@ public class PurgeDaoTest { return db.getDbClient().ceTaskCharacteristicsDao().selectByTaskUuids(db.getSession(), Collections.singletonList(taskUuid)); } - private Optional<CeTaskInputDao.DataStream> selectTaskInput(String taskUuid) { + private Optional<DbInputStream> selectTaskInput(String taskUuid) { return db.getDbClient().ceTaskInputDao().selectData(db.getSession(), taskUuid); } diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/scannercache/ScannerCacheDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/scannercache/ScannerCacheDaoTest.java new file mode 100644 index 00000000000..c453021c042 --- /dev/null +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/scannercache/ScannerCacheDaoTest.java @@ -0,0 +1,100 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.scannercache; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.SQLException; +import org.apache.commons.io.IOUtils; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.api.utils.System2; +import org.sonar.db.DbInputStream; +import org.sonar.db.DbSession; +import org.sonar.db.DbTester; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ScannerCacheDaoTest { + @Rule + public DbTester dbTester = DbTester.create(System2.INSTANCE); + + private final DbSession dbSession = dbTester.getSession(); + private final ScannerCacheDao underTest = dbTester.getDbClient().scannerCacheDao(); + + @Test + public void insert_should_insert_in_db() throws IOException { + underTest.insert(dbSession, "branch1", stringToInputStream("test data")); + dbSession.commit(); + assertThat(dbTester.countRowsOfTable("scanner_cache")).isOne(); + assertThat(dataStreamToString(underTest.selectData(dbSession, "branch1"))).isEqualTo("test data"); + } + + @Test + public void select_returns_empty_if_entry_doesnt_exist() { + underTest.insert(dbSession, "branch1", stringToInputStream("test data")); + assertThat(underTest.selectData(dbSession, "branch2")).isNull(); + } + + @Test + public void remove_all_should_delete_all() { + underTest.insert(dbSession, "branch1", stringToInputStream("test data")); + underTest.insert(dbSession, "branch2", stringToInputStream("test data")); + + assertThat(dbTester.countRowsOfTable("scanner_cache")).isEqualTo(2); + underTest.removeAll(dbSession); + dbSession.commit(); + assertThat(dbTester.countRowsOfTable("scanner_cache")).isZero(); + } + + @Test + public void throw_illegalstateexception_when_sql_excpetion() throws SQLException { + var dbSession = mock(DbSession.class); + var connection = mock(Connection.class); + when(dbSession.getConnection()).thenReturn(connection); + when(connection.prepareStatement(anyString())).thenThrow(new SQLException()); + + assertThatThrownBy(() -> underTest.selectData(dbSession, "uuid")) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Fail to select cache for branch uuid"); + + assertThatThrownBy(() -> underTest.insert(dbSession, "uuid", mock(InputStream.class))) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Fail to insert cache for branch uuid"); + } + + private static String dataStreamToString(DbInputStream dbInputStream) throws IOException { + try (DbInputStream is = dbInputStream) { + return IOUtils.toString(is, StandardCharsets.UTF_8); + } + } + + private static InputStream stringToInputStream(String str) { + return new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8)); + } + +} diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v94/CreateScannerCacheTable.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v94/CreateScannerCacheTable.java new file mode 100644 index 00000000000..12dc8d2179b --- /dev/null +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v94/CreateScannerCacheTable.java @@ -0,0 +1,44 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.platform.db.migration.version.v94; + +import java.sql.SQLException; +import org.sonar.db.Database; +import org.sonar.server.platform.db.migration.sql.CreateTableBuilder; +import org.sonar.server.platform.db.migration.step.CreateTableChange; + +import static org.sonar.server.platform.db.migration.def.BlobColumnDef.newBlobColumnDefBuilder; +import static org.sonar.server.platform.db.migration.def.VarcharColumnDef.UUID_SIZE; +import static org.sonar.server.platform.db.migration.def.VarcharColumnDef.newVarcharColumnDefBuilder; + +public class CreateScannerCacheTable extends CreateTableChange { + + public CreateScannerCacheTable(Database db) { + super(db, "scanner_cache"); + } + + @Override + public void execute(Context context, String tableName) throws SQLException { + context.execute(new CreateTableBuilder(getDialect(), tableName) + .addPkColumn(newVarcharColumnDefBuilder().setColumnName("branch_uuid").setIsNullable(false).setLimit(UUID_SIZE).build()) + .addColumn(newBlobColumnDefBuilder().setColumnName("data").setIsNullable(false).build()) + .build()); + } +} diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v94/DbVersion94.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v94/DbVersion94.java index 2d72af9b026..95cfbb8e30c 100644 --- a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v94/DbVersion94.java +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v94/DbVersion94.java @@ -29,7 +29,7 @@ public class DbVersion94 implements DbVersion { .add(6301, "Drop unused Issues Column REPORTER", DropReporterIssueColumn.class) .add(6302, "Drop unused Issues Column ACTION_PLAN_KEY", DropActionPlanKeyIssueColumn.class) .add(6303, "Drop unused Issues Column ISSUE_ATTRIBUTES", DropIssuesAttributesIssueColumn.class) - + .add(6304, "Create table 'SCANNER_CACHE", CreateScannerCacheTable.class) ; } } diff --git a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v94/CreateScannerCacheTableTest.java b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v94/CreateScannerCacheTableTest.java new file mode 100644 index 00000000000..393ee250bab --- /dev/null +++ b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v94/CreateScannerCacheTableTest.java @@ -0,0 +1,54 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.platform.db.migration.version.v94; + +import java.sql.SQLException; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.db.CoreDbTester; + +public class CreateScannerCacheTableTest { + private static final String TABLE_NAME = "scanner_cache"; + + @Rule + public final CoreDbTester db = CoreDbTester.createEmpty(); + + private final CreateScannerCacheTable underTest = new CreateScannerCacheTable(db.database()); + + @Test + public void migration_should_create_table() throws SQLException { + db.assertTableDoesNotExist(TABLE_NAME); + + underTest.execute(); + + db.assertTableExists(TABLE_NAME); + } + + @Test + public void migration_should_be_reentrant() throws SQLException { + db.assertTableDoesNotExist(TABLE_NAME); + + underTest.execute(); + //re-entrant + underTest.execute(); + + db.assertTableExists(TABLE_NAME); + } +} |