Browse Source

SONAR-16115 Store plugin's scanner cache in SonarQube

tags/9.4.0.54424
Duarte Meneses 2 years ago
parent
commit
1269984e8e
35 changed files with 802 additions and 44 deletions
  1. 4
    0
      server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReader.java
  2. 8
    0
      server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderImpl.java
  3. 4
    4
      server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ExtractReportStep.java
  4. 64
    0
      server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistPluginCacheStep.java
  5. 1
    0
      server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java
  6. 16
    0
      server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderRule.java
  7. 69
    0
      server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistPluginCacheStepTest.java
  8. 1
    0
      server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java
  9. 2
    0
      server/sonar-db-dao/src/main/java/org/sonar/db/DaoModule.java
  10. 8
    1
      server/sonar-db-dao/src/main/java/org/sonar/db/DbClient.java
  11. 46
    0
      server/sonar-db-dao/src/main/java/org/sonar/db/DbInputStream.java
  12. 2
    0
      server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java
  13. 4
    27
      server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskInputDao.java
  14. 7
    0
      server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java
  15. 1
    0
      server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java
  16. 82
    0
      server/sonar-db-dao/src/main/java/org/sonar/db/scannercache/ScannerCacheDao.java
  17. 28
    0
      server/sonar-db-dao/src/main/java/org/sonar/db/scannercache/ScannerCacheMapper.java
  18. 4
    0
      server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml
  19. 15
    0
      server/sonar-db-dao/src/main/resources/org/sonar/db/scannercache/ScannerCacheMapper.xml
  20. 6
    0
      server/sonar-db-dao/src/schema/schema-sq.ddl
  21. 7
    7
      server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeTaskInputDaoTest.java
  22. 2
    2
      server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java
  23. 100
    0
      server/sonar-db-dao/src/test/java/org/sonar/db/scannercache/ScannerCacheDaoTest.java
  24. 44
    0
      server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v94/CreateScannerCacheTable.java
  25. 1
    1
      server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v94/DbVersion94.java
  26. 54
    0
      server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v94/CreateScannerCacheTableTest.java
  27. 17
    0
      sonar-core/src/main/java/org/sonar/core/util/Protobuf.java
  28. 17
    2
      sonar-core/src/test/java/org/sonar/core/util/ProtobufTest.java
  29. 51
    0
      sonar-scanner-engine/src/main/java/org/sonar/scanner/report/PluginCachePublisher.java
  30. 81
    0
      sonar-scanner-engine/src/test/java/org/sonar/scanner/report/PluginCachePublisherTest.java
  31. 4
    0
      sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/FileStructure.java
  32. 17
    0
      sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportReader.java
  33. 7
    0
      sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportWriter.java
  34. 8
    0
      sonar-scanner-protocol/src/main/protobuf/scanner_internal.proto
  35. 20
    0
      sonar-scanner-protocol/src/test/java/org/sonar/scanner/protocol/output/ScannerReportReaderTest.java

+ 4
- 0
server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReader.java View File

@@ -19,6 +19,7 @@
*/
package org.sonar.ce.task.projectanalysis.batch;

import java.io.InputStream;
import java.util.Optional;
import javax.annotation.CheckForNull;
import org.sonar.core.util.CloseableIterator;
@@ -27,6 +28,9 @@ import org.sonar.scanner.protocol.output.ScannerReport;
public interface BatchReportReader {
ScannerReport.Metadata readMetadata();

@CheckForNull
InputStream getPluginCache();

CloseableIterator<String> readScannerLogs();

CloseableIterator<ScannerReport.ActiveRule> readActiveRules();

+ 8
- 0
server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderImpl.java View File

@@ -21,6 +21,7 @@ package org.sonar.ce.task.projectanalysis.batch;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.NoSuchElementException;
import java.util.Optional;
@@ -61,6 +62,13 @@ public class BatchReportReaderImpl implements BatchReportReader {
return this.metadata;
}

@CheckForNull
@Override
public InputStream getPluginCache() {
ensureInitialized();
return delegate.getPluginCache();
}

@Override
public CloseableIterator<String> readScannerLogs() {
ensureInitialized();

+ 4
- 4
server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ExtractReportStep.java View File

@@ -32,9 +32,9 @@ import org.sonar.api.utils.log.Loggers;
import org.sonar.ce.task.CeTask;
import org.sonar.ce.task.projectanalysis.batch.MutableBatchReportDirectoryHolder;
import org.sonar.ce.task.step.ComputationStep;
import org.sonar.db.DbInputStream;
import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
import org.sonar.db.ce.CeTaskInputDao;
import org.sonar.process.FileUtils2;

import static org.sonar.core.util.FileUtils.humanReadableByteCountSI;
@@ -63,11 +63,11 @@ public class ExtractReportStep implements ComputationStep {
@Override
public void execute(ComputationStep.Context context) {
try (DbSession dbSession = dbClient.openSession(false)) {
Optional<CeTaskInputDao.DataStream> opt = dbClient.ceTaskInputDao().selectData(dbSession, task.getUuid());
Optional<DbInputStream> opt = dbClient.ceTaskInputDao().selectData(dbSession, task.getUuid());
if (opt.isPresent()) {
File unzippedDir = tempFolder.newDir();
try (CeTaskInputDao.DataStream reportStream = opt.get();
InputStream zipStream = new BufferedInputStream(reportStream.getInputStream())) {
try (DbInputStream reportStream = opt.get();
InputStream zipStream = new BufferedInputStream(reportStream)) {
ZipUtils.unzip(zipStream, unzippedDir);
} catch (IOException e) {
throw new IllegalStateException("Fail to extract report " + task.getUuid() + " from database", e);

+ 64
- 0
server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistPluginCacheStep.java View File

@@ -0,0 +1,64 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.ce.task.projectanalysis.step;

import java.io.IOException;
import java.io.InputStream;
import org.sonar.api.utils.log.Logger;
import org.sonar.api.utils.log.Loggers;
import org.sonar.ce.task.projectanalysis.batch.BatchReportReader;
import org.sonar.ce.task.projectanalysis.component.TreeRootHolder;
import org.sonar.ce.task.step.ComputationStep;
import org.sonar.db.DbClient;

public class PersistPluginCacheStep implements ComputationStep {
private static final Logger LOGGER = Loggers.get(PersistPluginCacheStep.class);
private final BatchReportReader reportReader;
private final DbClient dbClient;
private final TreeRootHolder treeRootHolder;

public PersistPluginCacheStep(BatchReportReader reportReader, DbClient dbClient, TreeRootHolder treeRootHolder) {
this.reportReader = reportReader;
this.dbClient = dbClient;
this.treeRootHolder = treeRootHolder;
}

@Override
public String getDescription() {
return "Persist scanner plugin cache";
}

@Override
public void execute(ComputationStep.Context context) {
InputStream pluginCacheStream = reportReader.getPluginCache();
if (pluginCacheStream != null) {
try (var dataStream = pluginCacheStream;
var dbSession = dbClient.openSession(false)) {
String branchUuid = treeRootHolder.getRoot().getUuid();
dbClient.scannerCacheDao().remove(dbSession, branchUuid);
dbClient.scannerCacheDao().insert(dbSession, branchUuid, dataStream);
dbSession.commit();
} catch (IOException e) {
LOGGER.error("Error in reading plugin cache", e);
}
}

}
}

+ 1
- 0
server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java View File

@@ -48,6 +48,7 @@ public class ReportComputationSteps extends AbstractComputationSteps {
BuildComponentTreeStep.class,
ValidateProjectStep.class,

PersistPluginCacheStep.class,
LoadQualityProfilesStep.class,

// load project related stuffs

+ 16
- 0
server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderRule.java View File

@@ -20,6 +20,8 @@
package org.sonar.ce.task.projectanalysis.batch;

import com.google.common.base.Preconditions;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -57,6 +59,7 @@ public class BatchReportReaderRule implements TestRule, BatchReportReader {
private Map<Integer, List<ScannerReport.LineSgnificantCode>> significantCode = new HashMap<>();
private Map<Integer, ScannerReport.ChangedLines> changedLines = new HashMap<>();
private List<ScannerReport.AnalysisWarning> analysisWarnings = Collections.emptyList();
private byte[] pluginCache;

@Override
public Statement apply(final Statement statement, Description description) {
@@ -106,6 +109,19 @@ public class BatchReportReaderRule implements TestRule, BatchReportReader {
return metadata;
}

@CheckForNull
@Override
public InputStream getPluginCache() {
if (pluginCache == null) {
return null;
}
return new ByteArrayInputStream(pluginCache);
}

public void setPluginCache(byte[] cache) {
this.pluginCache = cache;
}

public BatchReportReaderRule setMetadata(ScannerReport.Metadata metadata) {
this.metadata = metadata;
return this;

+ 69
- 0
server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistPluginCacheStepTest.java View File

@@ -0,0 +1,69 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.ce.task.projectanalysis.step;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import org.apache.commons.io.IOUtils;
import org.junit.Rule;
import org.junit.Test;
import org.sonar.ce.task.projectanalysis.batch.BatchReportReaderRule;
import org.sonar.ce.task.projectanalysis.component.Component;
import org.sonar.ce.task.projectanalysis.component.TreeRootHolderRule;
import org.sonar.ce.task.step.ComputationStep;
import org.sonar.db.DbClient;
import org.sonar.db.DbInputStream;
import org.sonar.db.DbTester;

import static java.nio.charset.StandardCharsets.UTF_8;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

public class PersistPluginCacheStepTest {
@Rule
public BatchReportReaderRule reader = new BatchReportReaderRule();
@Rule
public DbTester dbTester = DbTester.create();
private final DbClient client = dbTester.getDbClient();
private final TreeRootHolderRule treeRootHolder = new TreeRootHolderRule();
private final PersistPluginCacheStep step = new PersistPluginCacheStep(reader, dbTester.getDbClient(), treeRootHolder);

@Test
public void inserts_cache() throws IOException {
reader.setPluginCache("test".getBytes(UTF_8));

Component root = mock(Component.class);
when(root.getUuid()).thenReturn("branch");
treeRootHolder.setRoot(root);

step.execute(mock(ComputationStep.Context.class));
assertThat(dbTester.countRowsOfTable("scanner_cache")).isOne();
try (DbInputStream data = client.scannerCacheDao().selectData(dbTester.getSession(), "branch")) {
assertThat(IOUtils.toString(data, UTF_8)).isEqualTo("test");
}
}

@Test
public void updates_cache() throws IOException {
client.scannerCacheDao().insert(dbTester.getSession(), "branch", new ByteArrayInputStream("test".getBytes(UTF_8)));
inserts_cache();
}
}

+ 1
- 0
server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java View File

@@ -94,6 +94,7 @@ public final class SqTables {
"rules_parameters",
"rules_profiles",
"rule_repositories",
"scanner_cache",
"schema_migrations",
"session_tokens",
"snapshots",

+ 2
- 0
server/sonar-db-dao/src/main/java/org/sonar/db/DaoModule.java View File

@@ -77,6 +77,7 @@ import org.sonar.db.qualityprofile.QualityProfileDao;
import org.sonar.db.qualityprofile.QualityProfileExportDao;
import org.sonar.db.rule.RuleDao;
import org.sonar.db.rule.RuleRepositoryDao;
import org.sonar.db.scannercache.ScannerCacheDao;
import org.sonar.db.schemamigration.SchemaMigrationDao;
import org.sonar.db.source.FileSourceDao;
import org.sonar.db.user.GroupDao;
@@ -156,6 +157,7 @@ public class DaoModule extends Module {
RuleDao.class,
RuleRepositoryDao.class,
SamlMessageIdDao.class,
ScannerCacheDao.class,
SnapshotDao.class,
SchemaMigrationDao.class,
SessionTokensDao.class,

+ 8
- 1
server/sonar-db-dao/src/main/java/org/sonar/db/DbClient.java View File

@@ -57,8 +57,8 @@ import org.sonar.db.permission.template.PermissionTemplateCharacteristicDao;
import org.sonar.db.permission.template.PermissionTemplateDao;
import org.sonar.db.plugin.PluginDao;
import org.sonar.db.portfolio.PortfolioDao;
import org.sonar.db.project.ProjectDao;
import org.sonar.db.project.ProjectBadgeTokenDao;
import org.sonar.db.project.ProjectDao;
import org.sonar.db.property.InternalComponentPropertiesDao;
import org.sonar.db.property.InternalPropertiesDao;
import org.sonar.db.property.PropertiesDao;
@@ -77,6 +77,7 @@ import org.sonar.db.qualityprofile.QualityProfileDao;
import org.sonar.db.qualityprofile.QualityProfileExportDao;
import org.sonar.db.rule.RuleDao;
import org.sonar.db.rule.RuleRepositoryDao;
import org.sonar.db.scannercache.ScannerCacheDao;
import org.sonar.db.schemamigration.SchemaMigrationDao;
import org.sonar.db.source.FileSourceDao;
import org.sonar.db.user.GroupDao;
@@ -168,6 +169,7 @@ public class DbClient {
private final UserDismissedMessagesDao userDismissedMessagesDao;
private final ApplicationProjectsDao applicationProjectsDao;
private final ProjectBadgeTokenDao projectBadgeTokenDao;
private final ScannerCacheDao scannerCacheDao;

public DbClient(Database database, MyBatis myBatis, DBSessions dbSessions, Dao... daos) {
this.database = database;
@@ -248,6 +250,7 @@ public class DbClient {
samlMessageIdDao = getDao(map, SamlMessageIdDao.class);
userDismissedMessagesDao = getDao(map, UserDismissedMessagesDao.class);
applicationProjectsDao = getDao(map, ApplicationProjectsDao.class);
scannerCacheDao = getDao(map, ScannerCacheDao.class);
}

public DbSession openSession(boolean batch) {
@@ -547,4 +550,8 @@ public class DbClient {
public ProjectBadgeTokenDao projectBadgeTokenDao() {
return projectBadgeTokenDao;
}

public ScannerCacheDao scannerCacheDao() {
return scannerCacheDao;
}
}

+ 46
- 0
server/sonar-db-dao/src/main/java/org/sonar/db/DbInputStream.java View File

@@ -0,0 +1,46 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.db;

import java.io.InputStream;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import org.apache.commons.io.IOUtils;
import org.sonar.api.internal.apachecommons.io.input.ProxyInputStream;

public class DbInputStream extends ProxyInputStream {
private final PreparedStatement stmt;
private final ResultSet rs;
private final InputStream stream;

public DbInputStream(PreparedStatement stmt, ResultSet rs, InputStream stream) {
super(stream);
this.stmt = stmt;
this.rs = rs;
this.stream = stream;
}

@Override
public void close() {
IOUtils.closeQuietly(stream);
DatabaseUtils.closeQuietly(rs);
DatabaseUtils.closeQuietly(stmt);
}
}

+ 2
- 0
server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java View File

@@ -136,6 +136,7 @@ import org.sonar.db.rule.RuleDto;
import org.sonar.db.rule.RuleMapper;
import org.sonar.db.rule.RuleParamDto;
import org.sonar.db.rule.RuleRepositoryMapper;
import org.sonar.db.scannercache.ScannerCacheMapper;
import org.sonar.db.schemamigration.SchemaMigrationDto;
import org.sonar.db.schemamigration.SchemaMigrationMapper;
import org.sonar.db.source.FileSourceMapper;
@@ -299,6 +300,7 @@ public class MyBatis {
RuleMapper.class,
RuleRepositoryMapper.class,
SamlMessageIdMapper.class,
ScannerCacheMapper.class,
SchemaMigrationMapper.class,
SessionTokenMapper.class,
SnapshotMapper.class,

+ 4
- 27
server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskInputDao.java View File

@@ -27,9 +27,9 @@ import java.sql.SQLException;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import org.apache.commons.io.IOUtils;
import org.sonar.api.utils.System2;
import org.sonar.db.Dao;
import org.sonar.db.DbInputStream;
import org.sonar.db.DatabaseUtils;
import org.sonar.db.DbSession;

@@ -57,16 +57,16 @@ public class CeTaskInputDao implements Dao {
}
}

public Optional<DataStream> selectData(DbSession dbSession, String taskUuid) {
public Optional<DbInputStream> selectData(DbSession dbSession, String taskUuid) {
PreparedStatement stmt = null;
ResultSet rs = null;
DataStream result = null;
DbInputStream result = null;
try {
stmt = dbSession.getConnection().prepareStatement("SELECT input_data FROM ce_task_input WHERE task_uuid=? AND input_data IS NOT NULL");
stmt.setString(1, taskUuid);
rs = stmt.executeQuery();
if (rs.next()) {
result = new DataStream(stmt, rs, rs.getBinaryStream(1));
result = new DbInputStream(stmt, rs, rs.getBinaryStream(1));
return Optional.of(result);
}
return Optional.empty();
@@ -88,27 +88,4 @@ public class CeTaskInputDao implements Dao {
CeTaskInputMapper mapper = dbSession.getMapper(CeTaskInputMapper.class);
DatabaseUtils.executeLargeUpdates(uuids, mapper::deleteByUuids);
}

public static class DataStream implements AutoCloseable {
private final PreparedStatement stmt;
private final ResultSet rs;
private final InputStream stream;

private DataStream(PreparedStatement stmt, ResultSet rs, InputStream stream) {
this.stmt = stmt;
this.rs = rs;
this.stream = stream;
}

public InputStream getInputStream() {
return stream;
}

@Override
public void close() {
IOUtils.closeQuietly(stream);
DatabaseUtils.closeQuietly(rs);
DatabaseUtils.closeQuietly(stmt);
}
}
}

+ 7
- 0
server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java View File

@@ -485,4 +485,11 @@ class PurgeCommands {
profiler.stop();
}

public void deleteScannerCache(String rootUuid) {
profiler.start("deleteScannerCache (scanner_cache)");
purgeMapper.deleteScannerCacheByBranchUuid(rootUuid);
session.commit();
profiler.stop();
}

}

+ 1
- 0
server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java View File

@@ -175,4 +175,5 @@ public interface PurgeMapper {

void deleteUserDismissedMessagesByProjectUuid(@Param("projectUuid") String projectUuid);

void deleteScannerCacheByBranchUuid(@Param("branchUuid") String branchUuid);
}

+ 82
- 0
server/sonar-db-dao/src/main/java/org/sonar/db/scannercache/ScannerCacheDao.java View File

@@ -0,0 +1,82 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.db.scannercache;

import java.io.InputStream;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import javax.annotation.CheckForNull;
import org.sonar.db.Dao;
import org.sonar.db.DbInputStream;
import org.sonar.db.DatabaseUtils;
import org.sonar.db.DbSession;

public class ScannerCacheDao implements Dao {
public void removeAll(DbSession session) {
mapper(session).removeAll();
}

public void remove(DbSession session, String branchUuid) {
mapper(session).remove(branchUuid);
}

public void insert(DbSession dbSession, String branchUuid, InputStream data) {
Connection connection = dbSession.getConnection();
try (PreparedStatement stmt = connection.prepareStatement(
"INSERT INTO scanner_cache (branch_uuid, data) VALUES (?, ?)")) {
stmt.setString(1, branchUuid);
stmt.setBinaryStream(2, data);
stmt.executeUpdate();
connection.commit();
} catch (SQLException e) {
throw new IllegalStateException("Fail to insert cache for branch " + branchUuid, e);
}
}

@CheckForNull
public DbInputStream selectData(DbSession dbSession, String branchUuid) {
PreparedStatement stmt = null;
ResultSet rs = null;
DbInputStream result = null;
try {
stmt = dbSession.getConnection().prepareStatement("SELECT data FROM scanner_cache WHERE branch_uuid=?");
stmt.setString(1, branchUuid);
rs = stmt.executeQuery();
if (rs.next()) {
result = new DbInputStream(stmt, rs, rs.getBinaryStream(1));
return result;
}
return null;
} catch (SQLException e) {
throw new IllegalStateException("Fail to select cache for branch " + branchUuid, e);
} finally {
if (result == null) {
DatabaseUtils.closeQuietly(rs);
DatabaseUtils.closeQuietly(stmt);
}
}
}

private static ScannerCacheMapper mapper(DbSession session) {
return session.getMapper(ScannerCacheMapper.class);
}
}

+ 28
- 0
server/sonar-db-dao/src/main/java/org/sonar/db/scannercache/ScannerCacheMapper.java View File

@@ -0,0 +1,28 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.db.scannercache;

import org.apache.ibatis.annotations.Param;

public interface ScannerCacheMapper {
void removeAll();

void remove(@Param("branchUuid") String branchUuid);
}

+ 4
- 0
server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml View File

@@ -592,5 +592,9 @@
<delete id="deleteUserDismissedMessagesByProjectUuid">
delete from user_dismissed_messages where project_uuid = #{projectUuid,jdbcType=VARCHAR}
</delete>

<delete id="deleteScannerCacheByBranchUuid">
delete from scanner_cache where branch_uuid = #{branchUuid,jdbcType=VARCHAR}
</delete>
</mapper>


+ 15
- 0
server/sonar-db-dao/src/main/resources/org/sonar/db/scannercache/ScannerCacheMapper.xml View File

@@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8" ?>

<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "mybatis-3-mapper.dtd">

<mapper namespace="org.sonar.db.scannercache.ScannerCacheMapper">
<delete id="removeAll">
delete from scanner_cache
</delete>

<delete id="remove">
delete from scanner_cache where branch_uuid = #{branchUuid,jdbcType=VARCHAR}
</delete>

</mapper>


+ 6
- 0
server/sonar-db-dao/src/schema/schema-sq.ddl View File

@@ -896,6 +896,12 @@ CREATE TABLE "SAML_MESSAGE_IDS"(
ALTER TABLE "SAML_MESSAGE_IDS" ADD CONSTRAINT "PK_SAML_MESSAGE_IDS" PRIMARY KEY("UUID");
CREATE UNIQUE INDEX "SAML_MESSAGE_IDS_UNIQUE" ON "SAML_MESSAGE_IDS"("MESSAGE_ID" NULLS FIRST);

CREATE TABLE "SCANNER_CACHE"(
"BRANCH_UUID" CHARACTER VARYING(40) NOT NULL,
"DATA" BINARY LARGE OBJECT NOT NULL
);
ALTER TABLE "SCANNER_CACHE" ADD CONSTRAINT "PK_SCANNER_CACHE" PRIMARY KEY("BRANCH_UUID");

CREATE TABLE "SESSION_TOKENS"(
"UUID" CHARACTER VARYING(40) NOT NULL,
"USER_UUID" CHARACTER VARYING(255) NOT NULL,

+ 7
- 7
server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeTaskInputDaoTest.java View File

@@ -25,8 +25,10 @@ import org.apache.commons.io.IOUtils;
import org.junit.Rule;
import org.junit.Test;
import org.sonar.api.utils.System2;
import org.sonar.db.DbInputStream;
import org.sonar.db.DbTester;

import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Collections.singleton;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -52,12 +54,10 @@ public class CeTaskInputDaoTest {
InputStream report = IOUtils.toInputStream(SOME_DATA);
underTest.insert(dbTester.getSession(), A_UUID, report);

Optional<CeTaskInputDao.DataStream> result = underTest.selectData(dbTester.getSession(), A_UUID);
Optional<DbInputStream> result = underTest.selectData(dbTester.getSession(), A_UUID);
assertThat(result).isPresent();
try {
assertThat(IOUtils.toString(result.get().getInputStream())).isEqualTo(SOME_DATA);
} finally {
result.get().close();
try (DbInputStream is = result.get()) {
assertThat(IOUtils.toString(is, UTF_8)).isEqualTo(SOME_DATA);
}
}

@@ -69,7 +69,7 @@ public class CeTaskInputDaoTest {

@Test
public void selectData_returns_absent_if_uuid_not_found() {
Optional<CeTaskInputDao.DataStream> result = underTest.selectData(dbTester.getSession(), A_UUID);
Optional<DbInputStream> result = underTest.selectData(dbTester.getSession(), A_UUID);
assertThat(result).isNotPresent();
}

@@ -78,7 +78,7 @@ public class CeTaskInputDaoTest {
insertData(A_UUID);
dbTester.commit();

Optional<CeTaskInputDao.DataStream> result = underTest.selectData(dbTester.getSession(), A_UUID);
Optional<DbInputStream> result = underTest.selectData(dbTester.getSession(), A_UUID);
assertThat(result).isNotPresent();
}


+ 2
- 2
server/sonar-db-dao/src/test/java/org/sonar/db/purge/PurgeDaoTest.java View File

@@ -44,6 +44,7 @@ import org.sonar.api.utils.System2;
import org.sonar.core.util.CloseableIterator;
import org.sonar.core.util.UuidFactoryFast;
import org.sonar.core.util.Uuids;
import org.sonar.db.DbInputStream;
import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
import org.sonar.db.DbTester;
@@ -52,7 +53,6 @@ import org.sonar.db.ce.CeActivityDto;
import org.sonar.db.ce.CeQueueDto;
import org.sonar.db.ce.CeQueueDto.Status;
import org.sonar.db.ce.CeTaskCharacteristicDto;
import org.sonar.db.ce.CeTaskInputDao;
import org.sonar.db.ce.CeTaskMessageDto;
import org.sonar.db.ce.CeTaskMessageType;
import org.sonar.db.ce.CeTaskTypes;
@@ -1675,7 +1675,7 @@ public class PurgeDaoTest {
return db.getDbClient().ceTaskCharacteristicsDao().selectByTaskUuids(db.getSession(), Collections.singletonList(taskUuid));
}

private Optional<CeTaskInputDao.DataStream> selectTaskInput(String taskUuid) {
private Optional<DbInputStream> selectTaskInput(String taskUuid) {
return db.getDbClient().ceTaskInputDao().selectData(db.getSession(), taskUuid);
}


+ 100
- 0
server/sonar-db-dao/src/test/java/org/sonar/db/scannercache/ScannerCacheDaoTest.java View File

@@ -0,0 +1,100 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.db.scannercache;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.sql.Connection;
import java.sql.SQLException;
import org.apache.commons.io.IOUtils;
import org.junit.Rule;
import org.junit.Test;
import org.sonar.api.utils.System2;
import org.sonar.db.DbInputStream;
import org.sonar.db.DbSession;
import org.sonar.db.DbTester;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

public class ScannerCacheDaoTest {
@Rule
public DbTester dbTester = DbTester.create(System2.INSTANCE);

private final DbSession dbSession = dbTester.getSession();
private final ScannerCacheDao underTest = dbTester.getDbClient().scannerCacheDao();

@Test
public void insert_should_insert_in_db() throws IOException {
underTest.insert(dbSession, "branch1", stringToInputStream("test data"));
dbSession.commit();
assertThat(dbTester.countRowsOfTable("scanner_cache")).isOne();
assertThat(dataStreamToString(underTest.selectData(dbSession, "branch1"))).isEqualTo("test data");
}

@Test
public void select_returns_empty_if_entry_doesnt_exist() {
underTest.insert(dbSession, "branch1", stringToInputStream("test data"));
assertThat(underTest.selectData(dbSession, "branch2")).isNull();
}

@Test
public void remove_all_should_delete_all() {
underTest.insert(dbSession, "branch1", stringToInputStream("test data"));
underTest.insert(dbSession, "branch2", stringToInputStream("test data"));

assertThat(dbTester.countRowsOfTable("scanner_cache")).isEqualTo(2);
underTest.removeAll(dbSession);
dbSession.commit();
assertThat(dbTester.countRowsOfTable("scanner_cache")).isZero();
}

@Test
public void throw_illegalstateexception_when_sql_excpetion() throws SQLException {
var dbSession = mock(DbSession.class);
var connection = mock(Connection.class);
when(dbSession.getConnection()).thenReturn(connection);
when(connection.prepareStatement(anyString())).thenThrow(new SQLException());

assertThatThrownBy(() -> underTest.selectData(dbSession, "uuid"))
.isInstanceOf(IllegalStateException.class)
.hasMessage("Fail to select cache for branch uuid");

assertThatThrownBy(() -> underTest.insert(dbSession, "uuid", mock(InputStream.class)))
.isInstanceOf(IllegalStateException.class)
.hasMessage("Fail to insert cache for branch uuid");
}

private static String dataStreamToString(DbInputStream dbInputStream) throws IOException {
try (DbInputStream is = dbInputStream) {
return IOUtils.toString(is, StandardCharsets.UTF_8);
}
}

private static InputStream stringToInputStream(String str) {
return new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8));
}

}

+ 44
- 0
server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v94/CreateScannerCacheTable.java View File

@@ -0,0 +1,44 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.platform.db.migration.version.v94;

import java.sql.SQLException;
import org.sonar.db.Database;
import org.sonar.server.platform.db.migration.sql.CreateTableBuilder;
import org.sonar.server.platform.db.migration.step.CreateTableChange;

import static org.sonar.server.platform.db.migration.def.BlobColumnDef.newBlobColumnDefBuilder;
import static org.sonar.server.platform.db.migration.def.VarcharColumnDef.UUID_SIZE;
import static org.sonar.server.platform.db.migration.def.VarcharColumnDef.newVarcharColumnDefBuilder;

public class CreateScannerCacheTable extends CreateTableChange {

public CreateScannerCacheTable(Database db) {
super(db, "scanner_cache");
}

@Override
public void execute(Context context, String tableName) throws SQLException {
context.execute(new CreateTableBuilder(getDialect(), tableName)
.addPkColumn(newVarcharColumnDefBuilder().setColumnName("branch_uuid").setIsNullable(false).setLimit(UUID_SIZE).build())
.addColumn(newBlobColumnDefBuilder().setColumnName("data").setIsNullable(false).build())
.build());
}
}

+ 1
- 1
server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v94/DbVersion94.java View File

@@ -29,7 +29,7 @@ public class DbVersion94 implements DbVersion {
.add(6301, "Drop unused Issues Column REPORTER", DropReporterIssueColumn.class)
.add(6302, "Drop unused Issues Column ACTION_PLAN_KEY", DropActionPlanKeyIssueColumn.class)
.add(6303, "Drop unused Issues Column ISSUE_ATTRIBUTES", DropIssuesAttributesIssueColumn.class)
.add(6304, "Create table 'SCANNER_CACHE", CreateScannerCacheTable.class)
;
}
}

+ 54
- 0
server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v94/CreateScannerCacheTableTest.java View File

@@ -0,0 +1,54 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.platform.db.migration.version.v94;

import java.sql.SQLException;
import org.junit.Rule;
import org.junit.Test;
import org.sonar.db.CoreDbTester;

public class CreateScannerCacheTableTest {
private static final String TABLE_NAME = "scanner_cache";

@Rule
public final CoreDbTester db = CoreDbTester.createEmpty();

private final CreateScannerCacheTable underTest = new CreateScannerCacheTable(db.database());

@Test
public void migration_should_create_table() throws SQLException {
db.assertTableDoesNotExist(TABLE_NAME);

underTest.execute();

db.assertTableExists(TABLE_NAME);
}

@Test
public void migration_should_be_reentrant() throws SQLException {
db.assertTableDoesNotExist(TABLE_NAME);

underTest.execute();
//re-entrant
underTest.execute();

db.assertTableExists(TABLE_NAME);
}
}

+ 17
- 0
sonar-core/src/main/java/org/sonar/core/util/Protobuf.java View File

@@ -29,6 +29,7 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.zip.GZIPOutputStream;
import org.apache.commons.io.IOUtils;

/**
@@ -82,6 +83,22 @@ public class Protobuf {
}
}

/**
* Writes a single message to {@code file}, compressed with gzip. Existing content is replaced, the message is not
* appended.
*/
public static void writeGzip(Message message, File toFile) {
OutputStream out = null;
try {
out = new GZIPOutputStream(new BufferedOutputStream(new FileOutputStream(toFile, false)));
message.writeTo(out);
} catch (Exception e) {
throw ContextException.of("Unable to write message", e).addContext("file", toFile);
} finally {
IOUtils.closeQuietly(out);
}
}

/**
* Streams multiple messages to {@code file}. Reading the messages back requires to
* call methods {@code readStream(...)}.

+ 17
- 2
sonar-core/src/test/java/org/sonar/core/util/ProtobufTest.java View File

@@ -20,6 +20,10 @@
package org.sonar.core.util;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.zip.GZIPInputStream;
import org.apache.commons.io.FileUtils;
import org.junit.Rule;
import org.junit.Test;
@@ -70,7 +74,7 @@ public class ProtobufTest {
}

@Test
public void fail_to_write_single_message() throws Exception {
public void fail_to_write_single_message() {
assertThatThrownBy(() -> {
File dir = temp.newFolder();
Protobuf.write(Fake.getDefaultInstance(), dir);
@@ -97,7 +101,18 @@ public class ProtobufTest {
}

@Test
public void fail_to_read_stream() throws Exception {
public void write_gzip_file() throws IOException {
File file = temp.newFile();

Fake item1 = Fake.newBuilder().setLabel("one").setLine(1).build();
Protobuf.writeGzip(item1, file);
try (InputStream is = new GZIPInputStream(new FileInputStream(file))) {
assertThat(Protobuf.read(is, Fake.parser()).getLabel()).isEqualTo("one");
}
}

@Test
public void fail_to_read_stream() {
assertThatThrownBy(() -> {
File dir = temp.newFolder();
Protobuf.readStream(dir, Fake.parser());

+ 51
- 0
sonar-scanner-engine/src/main/java/org/sonar/scanner/report/PluginCachePublisher.java View File

@@ -0,0 +1,51 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.scanner.report;

import com.google.protobuf.ByteString;
import java.util.Map;
import org.sonar.scanner.cache.PluginCacheEnabled;
import org.sonar.scanner.cache.ScannerWriteCache;
import org.sonar.scanner.protocol.internal.ScannerInternal.PluginCacheMsg;
import org.sonar.scanner.protocol.output.ScannerReportWriter;

public class PluginCachePublisher implements ReportPublisherStep {
private final PluginCacheEnabled pluginCacheEnabled;
private final ScannerWriteCache cache;

public PluginCachePublisher(PluginCacheEnabled pluginCacheEnabled, ScannerWriteCache cache) {
this.pluginCacheEnabled = pluginCacheEnabled;
this.cache = cache;
}

@Override
public void publish(ScannerReportWriter writer) {
if (!pluginCacheEnabled.isEnabled() || cache.getCache().isEmpty()) {
return;
}
PluginCacheMsg.Builder pluginCacheMsg = PluginCacheMsg.newBuilder();

for (Map.Entry<String, byte[]> entry : cache.getCache().entrySet()) {
pluginCacheMsg.putMap(entry.getKey(), ByteString.copyFrom(entry.getValue()));
}

writer.writePluginCache(pluginCacheMsg.build());
}
}

+ 81
- 0
sonar-scanner-engine/src/test/java/org/sonar/scanner/report/PluginCachePublisherTest.java View File

@@ -0,0 +1,81 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.scanner.report;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.sonar.scanner.cache.PluginCacheEnabled;
import org.sonar.scanner.cache.ScannerWriteCache;
import org.sonar.scanner.protocol.output.ScannerReportWriter;

import static java.util.Collections.emptyMap;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;

public class PluginCachePublisherTest {
@Rule
public TemporaryFolder temp = new TemporaryFolder();

private final ScannerWriteCache writeCache = mock(ScannerWriteCache.class);
private final PluginCacheEnabled pluginCacheEnabled = mock(PluginCacheEnabled.class);
private final PluginCachePublisher publisher = new PluginCachePublisher(pluginCacheEnabled, writeCache);

private ScannerReportWriter scannerReportWriter;

@Before
public void before() throws IOException {
scannerReportWriter = new ScannerReportWriter(temp.newFolder());
}

@Test
public void publish_does_nothing_if_cache_not_enabled() {
when(pluginCacheEnabled.isEnabled()).thenReturn(false);
publisher.publish(scannerReportWriter);
verifyNoInteractions(writeCache);
assertThat(scannerReportWriter.getFileStructure().root()).isEmptyDirectory();
}

@Test
public void publish_cache() {
when(writeCache.getCache()).thenReturn(Map.of("key1", "value1".getBytes(StandardCharsets.UTF_8)));
when(pluginCacheEnabled.isEnabled()).thenReturn(true);
publisher.publish(scannerReportWriter);
verify(writeCache, times(2)).getCache();
assertThat(scannerReportWriter.getFileStructure().pluginCache()).exists();
}

@Test
public void publish_empty_cache() {
when(writeCache.getCache()).thenReturn(emptyMap());
when(pluginCacheEnabled.isEnabled()).thenReturn(true);
publisher.publish(scannerReportWriter);
verify(writeCache).getCache();
assertThat(scannerReportWriter.getFileStructure().pluginCache()).doesNotExist();
}
}

+ 4
- 0
sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/FileStructure.java View File

@@ -66,6 +66,10 @@ public class FileStructure {
return new File(dir, "metadata.pb");
}

public File pluginCache() {
return new File(dir, "plugin-cache.pb");
}

public File analysisLog() {
return new File(dir, "analysis.log");
}

+ 17
- 0
sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportReader.java View File

@@ -19,7 +19,11 @@
*/
package org.sonar.scanner.protocol.output;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
import javax.annotation.CheckForNull;
import org.sonar.core.util.CloseableIterator;
import org.sonar.core.util.Protobuf;
@@ -75,6 +79,19 @@ public class ScannerReportReader {
return null;
}

@CheckForNull
public InputStream getPluginCache() {
File file = fileStructure.pluginCache();
if (fileExists(file)) {
try {
return new BufferedInputStream(new FileInputStream(fileStructure.pluginCache()));
} catch (FileNotFoundException e) {
throw new IllegalStateException("Unable to open file " + fileStructure.pluginCache(), e);
}
}
return null;
}

public ScannerReport.Component readComponent(int componentRef) {
File file = fileStructure.fileFor(FileStructure.Domain.COMPONENT, componentRef);
if (!fileExists(file)) {

+ 7
- 0
sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportWriter.java View File

@@ -26,6 +26,7 @@ import java.io.OutputStream;
import javax.annotation.concurrent.Immutable;
import org.sonar.core.util.ContextException;
import org.sonar.core.util.Protobuf;
import org.sonar.scanner.protocol.internal.ScannerInternal;

@Immutable
public class ScannerReportWriter {
@@ -94,6 +95,12 @@ public class ScannerReportWriter {
return file;
}

public File writePluginCache(ScannerInternal.PluginCacheMsg cacheMsg) {
File file = fileStructure.pluginCache();
Protobuf.writeGzip(cacheMsg, file);
return file;
}

public void appendComponentExternalIssue(int componentRef, ScannerReport.ExternalIssue issue) {
File file = fileStructure.fileFor(FileStructure.Domain.EXTERNAL_ISSUES, componentRef);
try (OutputStream out = new BufferedOutputStream(new FileOutputStream(file, true))) {

+ 8
- 0
sonar-scanner-protocol/src/main/protobuf/scanner_internal.proto View File

@@ -0,0 +1,8 @@
syntax = "proto3";

option java_package = "org.sonar.scanner.protocol.internal";
option optimize_for = SPEED;

message PluginCacheMsg {
map<string, bytes> map = 1;
}

+ 20
- 0
sonar-scanner-protocol/src/test/java/org/sonar/scanner/protocol/output/ScannerReportReaderTest.java View File

@@ -20,9 +20,13 @@
package org.sonar.scanner.protocol.output;

import com.google.common.collect.Lists;
import com.google.protobuf.ByteString;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.AbstractMap;
import java.util.zip.GZIPInputStream;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.junit.Before;
@@ -30,9 +34,12 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.sonar.core.util.CloseableIterator;
import org.sonar.core.util.Protobuf;
import org.sonar.scanner.protocol.internal.ScannerInternal.PluginCacheMsg;
import org.sonar.scanner.protocol.output.ScannerReport.Measure.StringValue;
import org.sonar.scanner.protocol.output.ScannerReport.SyntaxHighlightingRule.HighlightingType;

import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Arrays.asList;
import static java.util.Collections.singletonList;
import static org.assertj.core.api.Assertions.assertThat;
@@ -204,6 +211,19 @@ public class ScannerReportReaderTest {
assertThat(sut.readCpdTextBlocks(1)).toIterable().hasSize(1);
}

@Test
public void read_plugin_cache() throws IOException {
ScannerReportWriter writer = new ScannerReportWriter(dir);
writer.writePluginCache(PluginCacheMsg.newBuilder()
.putMap("key", ByteString.copyFrom("data", UTF_8))
.build());

ScannerReportReader reader = new ScannerReportReader(dir);

PluginCacheMsg cache = Protobuf.read(new GZIPInputStream(reader.getPluginCache()), PluginCacheMsg.parser());
assertThat(cache.getMapMap()).containsOnly(new AbstractMap.SimpleEntry<>("key", ByteString.copyFrom("data", UTF_8)));
}

@Test
public void empty_list_if_no_duplication_block_found() {
assertThat(underTest.readComponentDuplications(UNKNOWN_COMPONENT_REF)).toIterable().isEmpty();

Loading…
Cancel
Save