Browse Source

SONAR-18174 Analyzer cache should be kept in the file system to decrease memory use

tags/9.9.0.65466
Duarte Meneses 1 year ago
parent
commit
8b2213ef7a
48 changed files with 672 additions and 373 deletions
  1. 5
    2
      server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderImpl.java
  2. 3
    1
      server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderImplTest.java
  3. 2
    1
      server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/source/ReportIteratorTest.java
  4. 2
    1
      server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v99/DbVersion99.java
  5. 37
    0
      server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v99/DeleteAnalysisCache.java
  6. 74
    0
      server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v99/DeleteAnalysisCacheTest.java
  7. 5
    0
      server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v99/DeleteAnalysisCacheTest/schema.sql
  8. 20
    6
      sonar-core/src/main/java/org/sonar/core/util/Protobuf.java
  9. 20
    0
      sonar-core/src/test/java/org/sonar/core/util/ProtobufTest.java
  10. 2
    2
      sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheLoader.java
  11. 4
    4
      sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheMemoryStorage.java
  12. 7
    9
      sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheProvider.java
  13. 15
    9
      sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/DefaultAnalysisCacheLoader.java
  14. 1
    2
      sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/ScannerWriteCache.java
  15. 46
    31
      sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/WriteCacheImpl.java
  16. 4
    4
      sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/AnalysisResult.java
  17. 2
    21
      sonar-scanner-engine/src/main/java/org/sonar/scanner/report/AnalysisCachePublisher.java
  18. 5
    4
      sonar-scanner-engine/src/main/java/org/sonar/scanner/report/ReportPublisher.java
  19. 37
    0
      sonar-scanner-engine/src/main/java/org/sonar/scanner/report/ScannerFileStructureProvider.java
  20. 2
    0
      sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java
  21. 1
    1
      sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheEnabledTest.java
  22. 10
    6
      sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheMemoryStorageTest.java
  23. 31
    6
      sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheProviderTest.java
  24. 18
    17
      sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/DefaultAnalysisCacheLoaderTest.java
  25. 41
    12
      sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/WriteCacheImplTest.java
  26. 4
    2
      sonar-scanner-engine/src/test/java/org/sonar/scanner/cpd/CpdExecutorTest.java
  27. 2
    2
      sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/ScannerMediumTester.java
  28. 3
    1
      sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/scm/ScmMediumTest.java
  29. 4
    2
      sonar-scanner-engine/src/test/java/org/sonar/scanner/report/ActiveRulesPublisherTest.java
  30. 6
    44
      sonar-scanner-engine/src/test/java/org/sonar/scanner/report/AnalysisCachePublisherTest.java
  31. 12
    22
      sonar-scanner-engine/src/test/java/org/sonar/scanner/report/AnalysisContextReportPublisherTest.java
  32. 12
    11
      sonar-scanner-engine/src/test/java/org/sonar/scanner/report/AnalysisWarningsPublisherTest.java
  33. 5
    2
      sonar-scanner-engine/src/test/java/org/sonar/scanner/report/ChangedLinesPublisherTest.java
  34. 7
    6
      sonar-scanner-engine/src/test/java/org/sonar/scanner/report/ComponentsPublisherTest.java
  35. 23
    43
      sonar-scanner-engine/src/test/java/org/sonar/scanner/report/MetadataPublisherTest.java
  36. 8
    8
      sonar-scanner-engine/src/test/java/org/sonar/scanner/report/ReportPublisherTest.java
  37. 3
    1
      sonar-scanner-engine/src/test/java/org/sonar/scanner/report/SourcePublisherTest.java
  38. 3
    2
      sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/DefaultSensorStorageTest.java
  39. 39
    0
      sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/internal/SensorCacheData.java
  40. 24
    0
      sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/internal/package-info.java
  41. 1
    1
      sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/FileStructure.java
  42. 2
    3
      sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportReader.java
  43. 2
    12
      sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportWriter.java
  44. 25
    22
      sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/viewer/ScannerReportViewerApp.java
  45. 3
    2
      sonar-scanner-protocol/src/main/protobuf/scanner_internal.proto
  46. 41
    0
      sonar-scanner-protocol/src/test/java/org/sonar/scanner/protocol/internal/SensorCacheDataTest.java
  47. 45
    37
      sonar-scanner-protocol/src/test/java/org/sonar/scanner/protocol/output/ScannerReportReaderTest.java
  48. 4
    11
      sonar-scanner-protocol/src/test/java/org/sonar/scanner/protocol/output/ScannerReportWriterTest.java

+ 5
- 2
server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderImpl.java View File

@@ -31,15 +31,17 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.io.LineIterator;
import org.sonar.core.util.CloseableIterator;
import org.sonar.core.util.LineReaderIterator;
import org.sonar.scanner.protocol.output.FileStructure;
import org.sonar.scanner.protocol.output.ScannerReport;
import org.sonar.scanner.protocol.output.ScannerReport.LineSgnificantCode;
import org.sonar.scanner.protocol.output.ScannerReportReader;

import static java.nio.charset.StandardCharsets.UTF_8;

public class BatchReportReaderImpl implements BatchReportReader {

private final BatchReportDirectoryHolder batchReportDirectoryHolder;
private org.sonar.scanner.protocol.output.ScannerReportReader delegate;
private ScannerReportReader delegate;
// caching of metadata which are read often
private ScannerReport.Metadata metadata;

@@ -49,7 +51,8 @@ public class BatchReportReaderImpl implements BatchReportReader {

private void ensureInitialized() {
if (this.delegate == null) {
this.delegate = new org.sonar.scanner.protocol.output.ScannerReportReader(batchReportDirectoryHolder.getDirectory());
FileStructure fileStructure = new FileStructure(batchReportDirectoryHolder.getDirectory());
this.delegate = new ScannerReportReader(fileStructure);
}
}


+ 3
- 1
server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/batch/BatchReportReaderImplTest.java View File

@@ -28,6 +28,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.sonar.api.impl.utils.JUnitTempFolder;
import org.sonar.core.util.CloseableIterator;
import org.sonar.scanner.protocol.output.FileStructure;
import org.sonar.scanner.protocol.output.ScannerReport;
import org.sonar.scanner.protocol.output.ScannerReportWriter;

@@ -59,7 +60,8 @@ public class BatchReportReaderImplTest {
public void setUp() {
BatchReportDirectoryHolder holder = new ImmutableBatchReportDirectoryHolder(tempFolder.newDir());
underTest = new BatchReportReaderImpl(holder);
writer = new ScannerReportWriter(holder.getDirectory());
FileStructure fileStructure = new FileStructure(holder.getDirectory());
writer = new ScannerReportWriter(fileStructure);
}

@Test

+ 2
- 1
server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/source/ReportIteratorTest.java View File

@@ -45,7 +45,8 @@ public class ReportIteratorTest {
@Before
public void setUp() throws Exception {
File dir = temp.newFolder();
ScannerReportWriter writer = new ScannerReportWriter(dir);
FileStructure fileStructure = new FileStructure(dir);
ScannerReportWriter writer = new ScannerReportWriter(fileStructure);

writer.writeComponentCoverage(1, newArrayList(
ScannerReport.LineCoverage.newBuilder()

+ 2
- 1
server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v99/DbVersion99.java View File

@@ -26,6 +26,7 @@ public class DbVersion99 implements DbVersion {
@Override
public void addSteps(MigrationStepRegistry registry) {
registry
.add(6800, "Add node_name column to ce_activity table", AddNodeNameColumnToCeActivityTable.class);
.add(6800, "Add node_name column to ce_activity table", AddNodeNameColumnToCeActivityTable.class)
.add(6801, "Delete all analysis cache", DeleteAnalysisCache.class);
}
}

+ 37
- 0
server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v99/DeleteAnalysisCache.java View File

@@ -0,0 +1,37 @@
/*
* SonarQube
* Copyright (C) 2009-2023 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.platform.db.migration.version.v99;

import java.sql.SQLException;
import org.sonar.db.Database;
import org.sonar.server.platform.db.migration.step.DataChange;

public class DeleteAnalysisCache extends DataChange {
public DeleteAnalysisCache(Database db) {
super(db);
}

@Override
protected void execute(Context context) throws SQLException {
context.prepareUpsert("delete from scanner_analysis_cache")
.execute()
.commit();
}
}

+ 74
- 0
server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v99/DeleteAnalysisCacheTest.java View File

@@ -0,0 +1,74 @@
/*
* SonarQube
* Copyright (C) 2009-2023 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.platform.db.migration.version.v99;

import java.sql.SQLException;
import org.junit.Rule;
import org.junit.Test;
import org.sonar.db.CoreDbTester;

import static org.assertj.core.api.Assertions.assertThat;

public class DeleteAnalysisCacheTest {
@Rule
public final CoreDbTester db = CoreDbTester.createForSchema(DeleteAnalysisCacheTest.class, "schema.sql");

private final DeleteAnalysisCache underTest = new DeleteAnalysisCache(db.database());

@Test
public void no_op_if_no_data() throws SQLException {
assertThatCacheIsEmpty();
underTest.execute();
assertThatCacheIsEmpty();
}

@Test
public void deletes_all_data_in_table() throws SQLException {
insertCache("b1", "d1");
insertCache("b2", "d2");
assertThatCacheIsNotEmpty();
underTest.execute();
assertThatCacheIsEmpty();
}

@Test
public void migration_is_reentrant() throws SQLException {
insertCache("b1", "d1");
insertCache("b2", "d2");
assertThatCacheIsNotEmpty();
underTest.execute();
underTest.execute();
assertThatCacheIsEmpty();
}

private void assertThatCacheIsEmpty() {
assertThat(db.countRowsOfTable("scanner_analysis_cache")).isZero();
}

private void assertThatCacheIsNotEmpty() {
assertThat(db.countRowsOfTable("scanner_analysis_cache")).isNotZero();
}

private void insertCache(String branch, String data) {
db.executeInsert("scanner_analysis_cache",
"branch_uuid", branch,
"data", data);
}
}

+ 5
- 0
server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v99/DeleteAnalysisCacheTest/schema.sql View File

@@ -0,0 +1,5 @@
CREATE TABLE "SCANNER_ANALYSIS_CACHE"(
"BRANCH_UUID" CHARACTER VARYING(40) NOT NULL,
"DATA" BINARY LARGE OBJECT NOT NULL
);
ALTER TABLE "SCANNER_ANALYSIS_CACHE" ADD CONSTRAINT "PK_SCANNER_ANALYSIS_CACHE" PRIMARY KEY("BRANCH_UUID");

+ 20
- 6
sonar-core/src/main/java/org/sonar/core/util/Protobuf.java View File

@@ -29,6 +29,7 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import org.apache.commons.io.IOUtils;

@@ -136,9 +137,8 @@ public class Protobuf {
}

/**
* Reads a stream of messages. This method returns an empty iterator if there are no messages. An
* exception is raised on IO error, if file does not exist or if messages have a
* different type than {@code parser}.
* Reads a stream of messages. This method returns an empty iterator if there are no messages. An
* exception is raised on IO error, if file does not exist or if messages have a different type than {@code parser}.
*/
public static <MSG extends Message> CloseableIterator<MSG> readStream(File file, Parser<MSG> parser) {
try {
@@ -151,11 +151,25 @@ public class Protobuf {
}

/**
* Reads a stream of messages. This method returns an empty iterator if there are no messages. An
* Reads a stream of messages from a gzip file. This method returns an empty iterator if there are no messages. An
* exception is raised on IO error, if file does not exist or if messages have a different type than {@code parser}.
*/
public static <MSG extends Message> CloseableIterator<MSG> readGzipStream(File file, Parser<MSG> parser) {
try {
// the input stream is closed by the CloseableIterator
InputStream input = new GZIPInputStream(new BufferedInputStream(new FileInputStream(file)));
return readStream(input, parser);
} catch (Exception e) {
throw ContextException.of("Unable to read messages", e).addContext("file", file);
}
}

/**
* Reads a stream of messages. This method returns an empty iterator if there are no messages. An
* exception is raised on IO error or if messages have a different type than {@code parser}.
* <p>
* The stream is not closed by this method. It is closed when {@link CloseableIterator} traverses
* all messages or when {@link CloseableIterator#close()} is called.
* The stream is not closed by this method. It is closed when {@link CloseableIterator} traverses
* all messages or when {@link CloseableIterator#close()} is called.
* </p>
*/
public static <MSG extends Message> CloseableIterator<MSG> readStream(InputStream input, Parser<MSG> parser) {

+ 20
- 0
sonar-core/src/test/java/org/sonar/core/util/ProtobufTest.java View File

@@ -21,9 +21,12 @@ package org.sonar.core.util;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import org.apache.commons.io.FileUtils;
import org.junit.Rule;
import org.junit.Test;
@@ -111,6 +114,23 @@ public class ProtobufTest {
}
}

@Test
public void read_gzip_stream() throws IOException {
File file = temp.newFile();

Fake item1 = Fake.newBuilder().setLabel("one").setLine(1).build();
Fake item2 = Fake.newBuilder().setLabel("two").setLine(2).build();

try (OutputStream os = new GZIPOutputStream(new FileOutputStream(file))) {
item1.writeDelimitedTo(os);
item2.writeDelimitedTo(os);
}

Iterable<Fake> it = () -> Protobuf.readGzipStream(file, Fake.parser());

assertThat(it).containsExactly(item1, item2);
}

@Test
public void fail_to_read_stream() {
assertThatThrownBy(() -> {

+ 2
- 2
sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheLoader.java View File

@@ -20,8 +20,8 @@
package org.sonar.scanner.cache;

import java.util.Optional;
import org.sonar.scanner.protocol.internal.ScannerInternal;
import org.sonar.scanner.protocol.internal.SensorCacheData;

public interface AnalysisCacheLoader {
Optional<ScannerInternal.AnalysisCacheMsg> load();
Optional<SensorCacheData> load();
}

+ 4
- 4
sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheMemoryStorage.java View File

@@ -21,12 +21,12 @@ package org.sonar.scanner.cache;

import java.io.InputStream;
import javax.annotation.Nullable;
import org.sonar.scanner.protocol.internal.ScannerInternal.AnalysisCacheMsg;
import org.sonar.scanner.protocol.internal.SensorCacheData;

public class AnalysisCacheMemoryStorage implements AnalysisCacheStorage {
private final AnalysisCacheLoader loader;
@Nullable
private AnalysisCacheMsg cache;
private SensorCacheData cache;

public AnalysisCacheMemoryStorage(AnalysisCacheLoader loader) {
this.loader = loader;
@@ -37,7 +37,7 @@ public class AnalysisCacheMemoryStorage implements AnalysisCacheStorage {
if (!contains(key)) {
throw new IllegalArgumentException("Key not found: " + key);
}
return cache.getMapOrThrow(key).newInput();
return cache.getEntries().get(key).newInput();
}

@Override
@@ -45,7 +45,7 @@ public class AnalysisCacheMemoryStorage implements AnalysisCacheStorage {
if (cache == null) {
return false;
}
return cache.containsMap(key);
return cache.getEntries().containsKey(key);
}

public void load() {

+ 7
- 9
sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheProvider.java View File

@@ -20,15 +20,14 @@
package org.sonar.scanner.cache;

import java.io.InputStream;
import java.util.Map;
import org.jetbrains.annotations.Nullable;
import org.sonar.api.batch.sensor.cache.ReadCache;
import org.sonar.scanner.protocol.output.FileStructure;
import org.sonar.scanner.scan.branch.BranchConfiguration;
import org.springframework.context.annotation.Bean;

import static java.util.Collections.emptyMap;

public class AnalysisCacheProvider {

@Bean("ReadCache")
public ReadCache provideReader(AnalysisCacheEnabled analysisCacheEnabled, AnalysisCacheMemoryStorage storage) {
if (analysisCacheEnabled.isEnabled()) {
@@ -39,14 +38,13 @@ public class AnalysisCacheProvider {
}

@Bean("WriteCache")
public ScannerWriteCache provideWriter(AnalysisCacheEnabled analysisCacheEnabled, ReadCache readCache, BranchConfiguration branchConfiguration) {
if (analysisCacheEnabled.isEnabled()) {
return new WriteCacheImpl(readCache, branchConfiguration);
public ScannerWriteCache provideWriter(AnalysisCacheEnabled analysisCacheEnabled, ReadCache readCache, BranchConfiguration branchConfiguration, FileStructure fileStructure) {
if (analysisCacheEnabled.isEnabled() && !branchConfiguration.isPullRequest()) {
return new WriteCacheImpl(readCache, fileStructure);
}
return new NoOpWriteCache();
}


static class NoOpWriteCache implements ScannerWriteCache {
@Override
public void write(String s, InputStream inputStream) {
@@ -64,8 +62,8 @@ public class AnalysisCacheProvider {
}

@Override
public Map<String, byte[]> getCache() {
return emptyMap();
public void close() {
// no op
}
}


+ 15
- 9
sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/DefaultAnalysisCacheLoader.java View File

@@ -22,6 +22,8 @@ package org.sonar.scanner.cache;
import java.io.IOException;
import java.io.InputStream;
import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import java.util.zip.GZIPInputStream;
import org.sonar.api.scanner.fs.InputProject;
import org.sonar.api.utils.MessageException;
@@ -30,8 +32,8 @@ import org.sonar.api.utils.log.Loggers;
import org.sonar.api.utils.log.Profiler;
import org.sonar.core.util.Protobuf;
import org.sonar.scanner.bootstrap.DefaultScannerWsClient;
import org.sonar.scanner.protocol.internal.ScannerInternal;
import org.sonar.scanner.protocol.internal.ScannerInternal.AnalysisCacheMsg;
import org.sonar.scanner.protocol.internal.ScannerInternal.SensorCacheEntry;
import org.sonar.scanner.protocol.internal.SensorCacheData;
import org.sonar.scanner.scan.branch.BranchConfiguration;
import org.sonarqube.ws.client.GetRequest;
import org.sonarqube.ws.client.HttpException;
@@ -61,7 +63,7 @@ public class DefaultAnalysisCacheLoader implements AnalysisCacheLoader {
}

@Override
public Optional<AnalysisCacheMsg> load() {
public Optional<SensorCacheData> load() {
String url = URL + "?project=" + project.key();
if (branchConfiguration.referenceBranchName() != null) {
url = url + "&branch=" + branchConfiguration.referenceBranchName();
@@ -75,13 +77,13 @@ public class DefaultAnalysisCacheLoader implements AnalysisCacheLoader {
Optional<Integer> length = response.header(CONTENT_LENGTH).map(Integer::parseInt);
boolean hasGzipEncoding = contentEncoding.isPresent() && contentEncoding.get().equals("gzip");

AnalysisCacheMsg msg = hasGzipEncoding ? decompress(is) : Protobuf.read(is, AnalysisCacheMsg.parser());
SensorCacheData cache = hasGzipEncoding ? decompress(is) : read(is);
if (length.isPresent()) {
profiler.stopInfo(LOG_MSG + String.format(" (%s)", humanReadableByteCountSI(length.get())));
} else {
profiler.stopInfo(LOG_MSG);
}
return Optional.of(msg);
return Optional.of(cache);
} catch (HttpException e) {
if (e.code() == 404) {
profiler.stopInfo(LOG_MSG + " (404)");
@@ -93,11 +95,15 @@ public class DefaultAnalysisCacheLoader implements AnalysisCacheLoader {
}
}

private static AnalysisCacheMsg decompress(InputStream is) {
public SensorCacheData decompress(InputStream is) throws IOException {
try (GZIPInputStream gzipInputStream = new GZIPInputStream(is)) {
return Protobuf.read(gzipInputStream, ScannerInternal.AnalysisCacheMsg.parser());
} catch (IOException e) {
throw new IllegalStateException("Failed to decompress analysis cache", e);
return read(gzipInputStream);
}
}

public SensorCacheData read(InputStream is) {
Iterable<SensorCacheEntry> it = () -> Protobuf.readStream(is, SensorCacheEntry.parser());
return new SensorCacheData(StreamSupport.stream(it.spliterator(), false).collect(Collectors.toList()));
}
}


+ 1
- 2
sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/ScannerWriteCache.java View File

@@ -19,9 +19,8 @@
*/
package org.sonar.scanner.cache;

import java.util.Map;
import org.sonar.api.batch.sensor.cache.WriteCache;

public interface ScannerWriteCache extends WriteCache {
Map<String, byte[]> getCache();
void close();
}

+ 46
- 31
sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/WriteCacheImpl.java View File

@@ -19,40 +19,40 @@
*/
package org.sonar.scanner.cache;

import com.google.protobuf.ByteString;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.io.OutputStream;
import java.util.HashSet;
import java.util.Set;
import java.util.zip.GZIPOutputStream;
import org.sonar.api.batch.sensor.cache.ReadCache;
import org.sonar.scanner.scan.branch.BranchConfiguration;
import org.sonar.scanner.protocol.internal.ScannerInternal.SensorCacheEntry;
import org.sonar.scanner.protocol.output.FileStructure;

import static java.util.Collections.unmodifiableMap;
import static org.sonar.api.utils.Preconditions.checkArgument;
import static org.sonar.api.utils.Preconditions.checkNotNull;

public class WriteCacheImpl implements ScannerWriteCache {
private final ReadCache readCache;
private final BranchConfiguration branchConfiguration;
private final Map<String, byte[]> cache = new HashMap<>();
private final Set<String> keys = new HashSet<>();
private final FileStructure fileStructure;

public WriteCacheImpl(ReadCache readCache, BranchConfiguration branchConfiguration) {
private OutputStream stream = null;

public WriteCacheImpl(ReadCache readCache, FileStructure fileStructure) {
this.readCache = readCache;
this.branchConfiguration = branchConfiguration;
this.fileStructure = fileStructure;
}

@Override
public void write(String key, InputStream data) {
checkNotNull(data);
checkKey(key);
if (branchConfiguration.isPullRequest()) {
return;
}
try {
byte[] arr = data.readAllBytes();
cache.put(key, arr);
write(key, data.readAllBytes());
} catch (IOException e) {
throw new IllegalStateException("Failed to read stream", e);
throw new IllegalStateException("Failed to read sensor write cache data", e);
}
}

@@ -60,33 +60,48 @@ public class WriteCacheImpl implements ScannerWriteCache {
public void write(String key, byte[] data) {
checkNotNull(data);
checkKey(key);
if (branchConfiguration.isPullRequest()) {
return;
try {
OutputStream out = getStream();
toProto(key, data).writeDelimitedTo(out);
keys.add(key);
} catch (IOException e) {
throw new IllegalStateException("Failed to write to sensor cache file", e);
}
}

private OutputStream getStream() throws IOException {
if (stream == null) {
stream = new GZIPOutputStream(new FileOutputStream(fileStructure.analysisCache()));
}
cache.put(key, Arrays.copyOf(data, data.length));
return stream;
}

@Override
public void copyFromPrevious(String key) {
checkArgument(readCache.contains(key), "Previous cache doesn't contain key '%s'", key);
checkKey(key);
if (branchConfiguration.isPullRequest()) {
return;
}
try {
cache.put(key, readCache.read(key).readAllBytes());
} catch (IOException e) {
throw new IllegalStateException("Failed to read plugin cache for key " + key, e);
}
write(key, readCache.read(key));
}

private static SensorCacheEntry toProto(String key, byte[] data) {
return SensorCacheEntry.newBuilder()
.setKey(key)
.setData(ByteString.copyFrom(data))
.build();
}

@Override
public Map<String, byte[]> getCache() {
return unmodifiableMap(cache);
public void close() {
if (stream != null) {
try {
stream.close();
} catch (IOException e) {
throw new IllegalStateException("Failed to close sensor cache file", e);
}
}
}

private void checkKey(String key) {
checkNotNull(key);
checkArgument(!cache.containsKey(key), "Cache already contains key '%s'", key);
checkArgument(!keys.contains(key), "Cache already contains key '%s'", key);
}
}

+ 4
- 4
sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/AnalysisResult.java View File

@@ -32,16 +32,16 @@ import org.sonar.api.batch.fs.InputComponent;
import org.sonar.api.batch.fs.InputFile;
import org.sonar.api.batch.fs.TextPointer;
import org.sonar.api.batch.fs.TextRange;
import org.sonar.api.batch.sensor.highlighting.TypeOfText;
import org.sonar.api.batch.fs.internal.DefaultInputComponent;
import org.sonar.api.batch.fs.internal.DefaultInputFile;
import org.sonar.api.batch.sensor.highlighting.TypeOfText;
import org.sonar.api.scanner.fs.InputProject;
import org.sonar.core.util.CloseableIterator;
import org.sonar.scanner.protocol.output.FileStructure;
import org.sonar.scanner.protocol.output.ScannerReport;
import org.sonar.scanner.protocol.output.ScannerReport.Component;
import org.sonar.scanner.protocol.output.ScannerReport.Symbol;
import org.sonar.scanner.protocol.output.ScannerReportReader;
import org.sonar.scanner.report.ReportPublisher;
import org.sonar.scanner.report.ScannerReportUtils;
import org.sonar.scanner.scan.SpringProjectScanContainer;
import org.sonar.scanner.scan.filesystem.InputComponentStore;
@@ -57,8 +57,8 @@ public class AnalysisResult implements AnalysisObserver {
@Override
public void analysisCompleted(SpringProjectScanContainer container) {
LOG.info("Store analysis results in memory for later assertions in medium test");
ReportPublisher reportPublisher = container.getComponentByType(ReportPublisher.class);
reader = new ScannerReportReader(reportPublisher.getReportDir().toFile());
FileStructure fileStructure = container.getComponentByType(FileStructure.class);
reader = new ScannerReportReader(fileStructure);
project = container.getComponentByType(InputProject.class);

storeFs(container);

+ 2
- 21
sonar-scanner-engine/src/main/java/org/sonar/scanner/report/AnalysisCachePublisher.java View File

@@ -19,37 +19,18 @@
*/
package org.sonar.scanner.report;

import com.google.protobuf.ByteString;
import java.util.Map;
import org.sonar.scanner.cache.AnalysisCacheEnabled;
import org.sonar.scanner.cache.ScannerWriteCache;
import org.sonar.scanner.protocol.internal.ScannerInternal;
import org.sonar.scanner.protocol.internal.ScannerInternal.AnalysisCacheMsg;
import org.sonar.scanner.protocol.output.ScannerReportWriter;
import org.sonar.scanner.scan.branch.BranchConfiguration;

public class AnalysisCachePublisher implements ReportPublisherStep {
private final AnalysisCacheEnabled analysisCacheEnabled;
private final BranchConfiguration branchConfiguration;
private final ScannerWriteCache cache;

public AnalysisCachePublisher(AnalysisCacheEnabled analysisCacheEnabled, BranchConfiguration branchConfiguration, ScannerWriteCache cache) {
this.analysisCacheEnabled = analysisCacheEnabled;
this.branchConfiguration = branchConfiguration;
public AnalysisCachePublisher(ScannerWriteCache cache) {
this.cache = cache;
}

@Override
public void publish(ScannerReportWriter writer) {
if (!analysisCacheEnabled.isEnabled() || branchConfiguration.isPullRequest() || cache.getCache().isEmpty()) {
return;
}
AnalysisCacheMsg.Builder analysisCacheMsg = ScannerInternal.AnalysisCacheMsg.newBuilder();

for (Map.Entry<String, byte[]> entry : cache.getCache().entrySet()) {
analysisCacheMsg.putMap(entry.getKey(), ByteString.copyFrom(entry.getValue()));
}

writer.writeAnalysisCache(analysisCacheMsg.build());
cache.close();
}
}

+ 5
- 4
sonar-scanner-engine/src/main/java/org/sonar/scanner/report/ReportPublisher.java View File

@@ -45,6 +45,7 @@ import org.sonar.api.utils.log.Loggers;
import org.sonar.scanner.bootstrap.DefaultScannerWsClient;
import org.sonar.scanner.bootstrap.GlobalAnalysisMode;
import org.sonar.scanner.fs.InputModuleHierarchy;
import org.sonar.scanner.protocol.output.FileStructure;
import org.sonar.scanner.protocol.output.ScannerReportReader;
import org.sonar.scanner.protocol.output.ScannerReportWriter;
import org.sonar.scanner.scan.ScanProperties;
@@ -94,7 +95,7 @@ public class ReportPublisher implements Startable {
public ReportPublisher(ScanProperties properties, DefaultScannerWsClient wsClient, Server server, AnalysisContextReportPublisher contextPublisher,
InputModuleHierarchy moduleHierarchy, GlobalAnalysisMode analysisMode, TempFolder temp, ReportPublisherStep[] publishers, BranchConfiguration branchConfiguration,
CeTaskReportDataHolder ceTaskReportDataHolder, AnalysisWarnings analysisWarnings,
JavaArchitectureInformationProvider javaArchitectureInformationProvider) {
JavaArchitectureInformationProvider javaArchitectureInformationProvider, FileStructure fileStructure) {
this.wsClient = wsClient;
this.server = server;
this.contextPublisher = contextPublisher;
@@ -105,11 +106,11 @@ public class ReportPublisher implements Startable {
this.branchConfiguration = branchConfiguration;
this.properties = properties;
this.ceTaskReportDataHolder = ceTaskReportDataHolder;
this.reportDir = moduleHierarchy.root().getWorkDir().resolve("scanner-report");
this.reportDir = fileStructure.root().toPath();
this.analysisWarnings = analysisWarnings;
this.javaArchitectureInformationProvider = javaArchitectureInformationProvider;
this.writer = new ScannerReportWriter(reportDir.toFile());
this.reader = new ScannerReportReader(reportDir.toFile());
this.writer = new ScannerReportWriter(fileStructure);
this.reader = new ScannerReportReader(fileStructure);
}

@Override

+ 37
- 0
sonar-scanner-engine/src/main/java/org/sonar/scanner/report/ScannerFileStructureProvider.java View File

@@ -0,0 +1,37 @@
/*
* SonarQube
* Copyright (C) 2009-2023 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.scanner.report;

import java.io.File;
import org.sonar.scanner.fs.InputModuleHierarchy;
import org.sonar.scanner.protocol.output.FileStructure;
import org.springframework.context.annotation.Bean;

public class ScannerFileStructureProvider {
@Bean
public FileStructure fileStructure(InputModuleHierarchy inputModuleHierarchy) {
File reportDir = inputModuleHierarchy.root().getWorkDir().resolve("scanner-report").toFile();

if (!reportDir.exists() && !reportDir.mkdirs()) {
throw new IllegalStateException("Unable to create directory: " + reportDir);
}
return new FileStructure(reportDir);
}
}

+ 2
- 0
sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java View File

@@ -91,6 +91,7 @@ import org.sonar.scanner.report.ContextPropertiesPublisher;
import org.sonar.scanner.report.JavaArchitectureInformationProvider;
import org.sonar.scanner.report.MetadataPublisher;
import org.sonar.scanner.report.ReportPublisher;
import org.sonar.scanner.report.ScannerFileStructureProvider;
import org.sonar.scanner.report.SourcePublisher;
import org.sonar.scanner.report.TestExecutionPublisher;
import org.sonar.scanner.repository.ContextPropertiesCache;
@@ -242,6 +243,7 @@ public class SpringProjectScanContainer extends SpringComponentContainer {
ScannerMetrics.class,
JavaArchitectureInformationProvider.class,
ReportPublisher.class,
ScannerFileStructureProvider.class,
AnalysisContextReportPublisher.class,
MetadataPublisher.class,
ActiveRulesPublisher.class,

+ 1
- 1
sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheEnabledTest.java View File

@@ -33,7 +33,7 @@ public class AnalysisCacheEnabledTest {
private final AnalysisCacheEnabled analysisCacheEnabled = new AnalysisCacheEnabled(configuration);

@Test
public void enabled_by_default() {
public void enabled_by_default_if_not_pr() {
assertThat(analysisCacheEnabled.isEnabled()).isTrue();
}


+ 10
- 6
sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheMemoryStorageTest.java View File

@@ -21,12 +21,14 @@ package org.sonar.scanner.cache;

import com.google.protobuf.ByteString;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Optional;
import org.apache.commons.io.IOUtils;
import org.junit.Test;
import org.sonar.scanner.protocol.internal.ScannerInternal.AnalysisCacheMsg;
import org.sonar.scanner.protocol.internal.ScannerInternal.SensorCacheEntry;
import org.sonar.scanner.protocol.internal.SensorCacheData;

import static java.nio.charset.StandardCharsets.UTF_8;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Mockito.mock;
@@ -39,19 +41,21 @@ public class AnalysisCacheMemoryStorageTest {

@Test
public void storage_loads_with_loader() throws IOException {
when(loader.load()).thenReturn(Optional.of(AnalysisCacheMsg.newBuilder()
.putMap("key1", ByteString.copyFrom("value1", StandardCharsets.UTF_8))
SensorCacheData data = new SensorCacheData(List.of(SensorCacheEntry.newBuilder()
.setKey("key1")
.setData(ByteString.copyFrom("value1", UTF_8))
.build()));
when(loader.load()).thenReturn(Optional.of(data));

storage.load();
verify(loader).load();
assertThat(IOUtils.toString(storage.get("key1"), StandardCharsets.UTF_8)).isEqualTo("value1");
assertThat(IOUtils.toString(storage.get("key1"), UTF_8)).isEqualTo("value1");
assertThat(storage.contains("key1")).isTrue();
}

@Test
public void get_throws_IAE_if_doesnt_contain_key() {
when(loader.load()).thenReturn(Optional.of(AnalysisCacheMsg.newBuilder().build()));
when(loader.load()).thenReturn(Optional.empty());
storage.load();
assertThat(storage.contains("key1")).isFalse();
assertThatThrownBy(() -> storage.get("key1")).isInstanceOf(IllegalArgumentException.class);

+ 31
- 6
sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheProviderTest.java View File

@@ -19,8 +19,15 @@
*/
package org.sonar.scanner.cache;

import java.io.IOException;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.sonar.api.batch.sensor.cache.ReadCache;
import org.sonar.scanner.cache.AnalysisCacheProvider.NoOpReadCache;
import org.sonar.scanner.cache.AnalysisCacheProvider.NoOpWriteCache;
import org.sonar.scanner.protocol.output.FileStructure;
import org.sonar.scanner.scan.branch.BranchConfiguration;

import static org.assertj.core.api.Assertions.assertThat;
@@ -29,25 +36,43 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

public class AnalysisCacheProviderTest {

private final AnalysisCacheEnabled analysisCacheEnabled = mock(AnalysisCacheEnabled.class);
private final AnalysisCacheMemoryStorage storage = mock(AnalysisCacheMemoryStorage.class);
private final ReadCache readCache = mock(ReadCache.class);
private final AnalysisCacheProvider cacheProvider = new AnalysisCacheProvider();
private final BranchConfiguration branchConfiguration = mock(BranchConfiguration.class);
private final AnalysisCacheProvider cacheProvider = new AnalysisCacheProvider();

@Rule
public TemporaryFolder temp = new TemporaryFolder();

private FileStructure fileStructure;

@Before
public void setup() throws IOException {
fileStructure = new FileStructure(temp.newFolder());
when(branchConfiguration.isPullRequest()).thenReturn(false);
}

@Test
public void provide_noop_writer_cache_if_pr() {
when(branchConfiguration.isPullRequest()).thenReturn(true);
when(analysisCacheEnabled.isEnabled()).thenReturn(true);
var cache = cacheProvider.provideWriter(analysisCacheEnabled, readCache, branchConfiguration, fileStructure);
assertThat(cache).isInstanceOf(AnalysisCacheProvider.NoOpWriteCache.class);
}

@Test
public void provide_noop_reader_cache_when_disable() {
when(analysisCacheEnabled.isEnabled()).thenReturn(false);
var cache = cacheProvider.provideReader(analysisCacheEnabled, storage);
assertThat(cache).isInstanceOf(AnalysisCacheProvider.NoOpReadCache.class);
assertThat(cache).isInstanceOf(NoOpReadCache.class);
}

@Test
public void provide_noop_writer_cache_when_disable() {
when(analysisCacheEnabled.isEnabled()).thenReturn(false);
var cache = cacheProvider.provideWriter(analysisCacheEnabled, readCache, branchConfiguration);
assertThat(cache).isInstanceOf(AnalysisCacheProvider.NoOpWriteCache.class);
var cache = cacheProvider.provideWriter(analysisCacheEnabled, readCache, branchConfiguration, fileStructure);
assertThat(cache).isInstanceOf(NoOpWriteCache.class);
}

@Test
@@ -61,7 +86,7 @@ public class AnalysisCacheProviderTest {
@Test
public void provide_real_writer_cache_when_enable() {
when(analysisCacheEnabled.isEnabled()).thenReturn(true);
var cache = cacheProvider.provideWriter(analysisCacheEnabled, readCache, branchConfiguration);
var cache = cacheProvider.provideWriter(analysisCacheEnabled, readCache, branchConfiguration, fileStructure);
assertThat(cache).isInstanceOf(WriteCacheImpl.class);
}
}

+ 18
- 17
sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/DefaultAnalysisCacheLoaderTest.java View File

@@ -26,8 +26,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Optional;
import java.util.zip.DeflaterInputStream;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import org.junit.Before;
import org.junit.Rule;
@@ -37,7 +35,8 @@ import org.sonar.api.scanner.fs.InputProject;
import org.sonar.api.utils.MessageException;
import org.sonar.api.utils.log.LogTester;
import org.sonar.scanner.bootstrap.DefaultScannerWsClient;
import org.sonar.scanner.protocol.internal.ScannerInternal.AnalysisCacheMsg;
import org.sonar.scanner.protocol.internal.ScannerInternal.SensorCacheEntry;
import org.sonar.scanner.protocol.internal.SensorCacheData;
import org.sonar.scanner.scan.branch.BranchConfiguration;
import org.sonarqube.ws.client.HttpException;
import org.sonarqube.ws.client.WsRequest;
@@ -45,6 +44,7 @@ import org.sonarqube.ws.client.WsResponse;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.assertj.core.api.Assertions.entry;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
@@ -52,8 +52,9 @@ import static org.mockito.Mockito.when;
import static org.sonar.scanner.cache.DefaultAnalysisCacheLoader.CONTENT_ENCODING;

public class DefaultAnalysisCacheLoaderTest {
private final static AnalysisCacheMsg MSG = AnalysisCacheMsg.newBuilder()
.putMap("key", ByteString.copyFrom("value", StandardCharsets.UTF_8))
private final static SensorCacheEntry MSG = SensorCacheEntry.newBuilder()
.setKey("key")
.setData(ByteString.copyFrom("value", StandardCharsets.UTF_8))
.build();
private final WsResponse response = mock(WsResponse.class);
private final DefaultScannerWsClient wsClient = mock(DefaultScannerWsClient.class);
@@ -73,8 +74,8 @@ public class DefaultAnalysisCacheLoaderTest {
public void loads_content_and_logs_size() throws IOException {
setResponse(MSG);
when(response.header("Content-Length")).thenReturn(Optional.of("123"));
AnalysisCacheMsg msg = loader.load().get();
assertThat(msg).isEqualTo(MSG);
SensorCacheData msg = loader.load().get();
assertThat(msg.getEntries()).containsOnly(entry(MSG.getKey(), MSG.getData()));
assertRequestPath("api/analysis_cache/get?project=myproject");
assertThat(logs.logs()).anyMatch(s -> s.startsWith("Load analysis cache (123 bytes)"));
}
@@ -84,9 +85,9 @@ public class DefaultAnalysisCacheLoaderTest {
when(branchConfiguration.referenceBranchName()).thenReturn("name");

setResponse(MSG);
AnalysisCacheMsg msg = loader.load().get();
SensorCacheData msg = loader.load().get();

assertThat(msg).isEqualTo(MSG);
assertThat(msg.getEntries()).containsOnly(entry(MSG.getKey(), MSG.getData()));
assertRequestPath("api/analysis_cache/get?project=myproject&branch=name");
assertThat(logs.logs()).anyMatch(s -> s.startsWith("Load analysis cache | time="));
}
@@ -94,8 +95,8 @@ public class DefaultAnalysisCacheLoaderTest {
@Test
public void loads_compressed_content() throws IOException {
setCompressedResponse(MSG);
AnalysisCacheMsg msg = loader.load().get();
assertThat(msg).isEqualTo(MSG);
SensorCacheData msg = loader.load().get();
assertThat(msg.getEntries()).containsOnly(entry(MSG.getKey(), MSG.getData()));
}

@Test
@@ -127,11 +128,11 @@ public class DefaultAnalysisCacheLoaderTest {
assertThat(requestCaptor.getValue().getPath()).isEqualTo(expectedPath);
}

private void setResponse(AnalysisCacheMsg msg) throws IOException {
private void setResponse(SensorCacheEntry msg) throws IOException {
when(response.contentStream()).thenReturn(createInputStream(msg));
}

private void setCompressedResponse(AnalysisCacheMsg msg) throws IOException {
private void setCompressedResponse(SensorCacheEntry msg) throws IOException {
when(response.contentStream()).thenReturn(createCompressedInputStream(msg));
when(response.header(CONTENT_ENCODING)).thenReturn(Optional.of("gzip"));
}
@@ -141,16 +142,16 @@ public class DefaultAnalysisCacheLoaderTest {
when(response.header(CONTENT_ENCODING)).thenReturn(Optional.of("gzip"));
}

private InputStream createInputStream(AnalysisCacheMsg analysisCacheMsg) throws IOException {
private InputStream createInputStream(SensorCacheEntry analysisCacheMsg) throws IOException {
ByteArrayOutputStream serialized = new ByteArrayOutputStream(analysisCacheMsg.getSerializedSize());
analysisCacheMsg.writeTo(serialized);
analysisCacheMsg.writeDelimitedTo(serialized);
return new ByteArrayInputStream(serialized.toByteArray());
}

private InputStream createCompressedInputStream(AnalysisCacheMsg analysisCacheMsg) throws IOException {
private InputStream createCompressedInputStream(SensorCacheEntry analysisCacheMsg) throws IOException {
ByteArrayOutputStream serialized = new ByteArrayOutputStream(analysisCacheMsg.getSerializedSize());
GZIPOutputStream compressed = new GZIPOutputStream(serialized);
analysisCacheMsg.writeTo(compressed);
analysisCacheMsg.writeDelimitedTo(compressed);
compressed.close();
return new ByteArrayInputStream(serialized.toByteArray());
}

+ 41
- 12
sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/WriteCacheImplTest.java View File

@@ -20,48 +20,68 @@
package org.sonar.scanner.cache;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.sonar.core.util.Protobuf;
import org.sonar.scanner.protocol.internal.ScannerInternal.SensorCacheEntry;
import org.sonar.scanner.protocol.output.FileStructure;
import org.sonar.scanner.scan.branch.BranchConfiguration;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.assertj.core.data.MapEntry.entry;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

public class WriteCacheImplTest {
private final ReadCacheImpl readCache = mock(ReadCacheImpl.class);
private final BranchConfiguration branchConfiguration = mock(BranchConfiguration.class);
private final WriteCacheImpl writeCache = new WriteCacheImpl(readCache, branchConfiguration);
private FileStructure fileStructure;
private WriteCacheImpl writeCache;

@Rule
public TemporaryFolder temp = new TemporaryFolder();

@Before
public void setUp() throws IOException {
fileStructure = new FileStructure(temp.newFolder());
writeCache = new WriteCacheImpl(readCache, fileStructure);
}

@Test
public void write_bytes_adds_entries() {
public void write_bytes_adds_entries() throws IOException {
byte[] b1 = new byte[] {1, 2, 3};
byte[] b2 = new byte[] {3, 4};
writeCache.write("key", b1);
writeCache.write("key2", b2);

assertThat(writeCache.getCache()).containsOnly(entry("key", b1), entry("key2", b2));
assertThatCacheContains(Map.of("key", b1, "key2", b2));
}

@Test
public void dont_write_if_its_pull_request() {
public void dont_write_if_its_pull_request() throws IOException {
byte[] b1 = new byte[] {1, 2, 3};
when(branchConfiguration.isPullRequest()).thenReturn(true);
writeCache.write("key1", b1);
writeCache.write("key2", new ByteArrayInputStream(b1));
assertThat(writeCache.getCache()).isEmpty();
assertThatCacheContains(Map.of());
}

@Test
public void write_inputStream_adds_entries() {
public void write_inputStream_adds_entries() throws IOException {
byte[] b1 = new byte[] {1, 2, 3};
byte[] b2 = new byte[] {3, 4};
writeCache.write("key", new ByteArrayInputStream(b1));
writeCache.write("key2", new ByteArrayInputStream(b2));

assertThat(writeCache.getCache()).containsOnly(entry("key", b1), entry("key2", b2));
assertThatCacheContains(Map.of("key", b1, "key2", b2));
}

@Test
@@ -69,7 +89,6 @@ public class WriteCacheImplTest {
byte[] b1 = new byte[] {1};
byte[] b2 = new byte[] {2};


writeCache.write("key", b1);
assertThatThrownBy(() -> writeCache.write("key", b2))
.isInstanceOf(IllegalArgumentException.class)
@@ -78,19 +97,29 @@ public class WriteCacheImplTest {

@Test
public void copyFromPrevious_throws_IAE_if_read_cache_doesnt_contain_key() {
assertThatThrownBy(() -> writeCache.copyFromPrevious("key"))
assertThatThrownBy(() -> writeCache.copyFromPrevious("key"))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Previous cache doesn't contain key 'key'");
}

@Test
public void copyFromPrevious_reads_from_readCache() {
public void copyFromPrevious_reads_from_readCache() throws IOException {
byte[] b = new byte[] {1};
InputStream value = new ByteArrayInputStream(b);
when(readCache.contains("key")).thenReturn(true);
when(readCache.read("key")).thenReturn(value);
writeCache.copyFromPrevious("key");

assertThat(writeCache.getCache()).containsOnly(entry("key", b));
assertThatCacheContains(Map.of("key", b));
}

private void assertThatCacheContains(Map<String, byte[]> expectedData) {
writeCache.close();
File cacheFile = fileStructure.analysisCache();
Iterable<SensorCacheEntry> it = () -> Protobuf.readGzipStream(cacheFile, SensorCacheEntry.parser());
Map<String, byte[]> data = StreamSupport.stream(it.spliterator(), false)
.collect(Collectors.toMap(SensorCacheEntry::getKey, e -> e.getData().toByteArray()));

assertThat(data).containsAllEntriesOf(expectedData);
}
}

+ 4
- 2
sonar-scanner-engine/src/test/java/org/sonar/scanner/cpd/CpdExecutorTest.java View File

@@ -47,6 +47,7 @@ import org.sonar.duplications.block.ByteArray;
import org.sonar.duplications.index.CloneGroup;
import org.sonar.duplications.index.ClonePart;
import org.sonar.scanner.cpd.index.SonarCpdBlockIndex;
import org.sonar.scanner.protocol.output.FileStructure;
import org.sonar.scanner.protocol.output.ScannerReport.Duplicate;
import org.sonar.scanner.protocol.output.ScannerReport.Duplication;
import org.sonar.scanner.protocol.output.ScannerReportReader;
@@ -84,13 +85,14 @@ public class CpdExecutorTest {
@Before
public void setUp() throws IOException {
File outputDir = temp.newFolder();
FileStructure fileStructure = new FileStructure(outputDir);
baseDir = temp.newFolder();
when(publisher.getWriter()).thenReturn(new ScannerReportWriter(outputDir));
when(publisher.getWriter()).thenReturn(new ScannerReportWriter(fileStructure));

DefaultInputProject project = TestInputFileBuilder.newDefaultInputProject("foo", baseDir);
componentStore = new InputComponentStore(mock(BranchConfiguration.class), sonarRuntime);
executor = new CpdExecutor(settings, index, publisher, componentStore, executorService);
reader = new ScannerReportReader(outputDir);
reader = new ScannerReportReader(fileStructure);

batchComponent1 = createComponent("src/Foo.php", 5);
batchComponent2 = createComponent("src/Foo2.php", 5);

+ 2
- 2
sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/ScannerMediumTester.java View File

@@ -59,7 +59,7 @@ import org.sonar.batch.bootstrapper.EnvironmentInformation;
import org.sonar.batch.bootstrapper.LogOutput;
import org.sonar.scanner.bootstrap.GlobalAnalysisMode;
import org.sonar.scanner.cache.AnalysisCacheLoader;
import org.sonar.scanner.protocol.internal.ScannerInternal;
import org.sonar.scanner.protocol.internal.SensorCacheData;
import org.sonar.scanner.report.CeTaskReportDataHolder;
import org.sonar.scanner.repository.FileData;
import org.sonar.scanner.repository.MetricsRepository;
@@ -525,7 +525,7 @@ public class ScannerMediumTester extends ExternalResource {
@Priority(1)
private static class FakeAnalysisCacheLoader implements AnalysisCacheLoader {
@Override
public Optional<ScannerInternal.AnalysisCacheMsg> load() {
public Optional<SensorCacheData> load() {
return Optional.empty();
}
}

+ 3
- 1
sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/scm/ScmMediumTest.java View File

@@ -35,6 +35,7 @@ import org.sonar.api.SonarEdition;
import org.sonar.api.utils.log.LogTester;
import org.sonar.scanner.mediumtest.ScannerMediumTester;
import org.sonar.scanner.mediumtest.ScannerMediumTester.AnalysisBuilder;
import org.sonar.scanner.protocol.output.FileStructure;
import org.sonar.scanner.protocol.output.ScannerReport;
import org.sonar.scanner.protocol.output.ScannerReport.Changesets.Changeset;
import org.sonar.scanner.protocol.output.ScannerReport.Component;
@@ -108,7 +109,8 @@ public class ScmMediumTest {

private ScannerReport.Changesets getChangesets(File baseDir, String path) {
File reportDir = new File(baseDir, ".sonar/scanner-report");
ScannerReportReader reader = new ScannerReportReader(reportDir);
FileStructure fileStructure = new FileStructure(reportDir);
ScannerReportReader reader = new ScannerReportReader(fileStructure);

Component project = reader.readComponent(reader.readMetadata().getRootComponentRef());
for (Integer fileRef : project.getChildRefList()) {

+ 4
- 2
sonar-scanner-engine/src/test/java/org/sonar/scanner/report/ActiveRulesPublisherTest.java View File

@@ -29,6 +29,7 @@ import org.sonar.api.batch.rule.internal.NewActiveRule;
import org.sonar.api.rule.RuleKey;
import org.sonar.core.util.CloseableIterator;
import org.sonar.scanner.protocol.Constants;
import org.sonar.scanner.protocol.output.FileStructure;
import org.sonar.scanner.protocol.output.ScannerReport;
import org.sonar.scanner.protocol.output.ScannerReportReader;
import org.sonar.scanner.protocol.output.ScannerReportWriter;
@@ -44,7 +45,8 @@ public class ActiveRulesPublisherTest {
@Test
public void write() throws Exception {
File outputDir = temp.newFolder();
ScannerReportWriter writer = new ScannerReportWriter(outputDir);
FileStructure fileStructure = new FileStructure(outputDir);
ScannerReportWriter writer = new ScannerReportWriter(fileStructure);

NewActiveRule ar = new NewActiveRule.Builder()
.setRuleKey(RuleKey.of("java", "S001"))
@@ -59,7 +61,7 @@ public class ActiveRulesPublisherTest {
ActiveRulesPublisher underTest = new ActiveRulesPublisher(activeRules);
underTest.publish(writer);

ScannerReportReader reader = new ScannerReportReader(outputDir);
ScannerReportReader reader = new ScannerReportReader(fileStructure);
try (CloseableIterator<ScannerReport.ActiveRule> readIt = reader.readActiveRules()) {
ScannerReport.ActiveRule reportAr = readIt.next();
assertThat(reportAr.getRuleRepository()).isEqualTo("java");

+ 6
- 44
sonar-scanner-engine/src/test/java/org/sonar/scanner/report/AnalysisCachePublisherTest.java View File

@@ -20,73 +20,35 @@
package org.sonar.scanner.report;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.sonar.scanner.cache.AnalysisCacheEnabled;
import org.sonar.scanner.cache.ScannerWriteCache;
import org.sonar.scanner.protocol.output.FileStructure;
import org.sonar.scanner.protocol.output.ScannerReportWriter;
import org.sonar.scanner.scan.branch.BranchConfiguration;

import static java.util.Collections.emptyMap;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;

public class AnalysisCachePublisherTest {
@Rule
public TemporaryFolder temp = new TemporaryFolder();

private final ScannerWriteCache writeCache = mock(ScannerWriteCache.class);
private final AnalysisCacheEnabled analysisCacheEnabled = mock(AnalysisCacheEnabled.class);
private final BranchConfiguration branchConfiguration = mock(BranchConfiguration.class);
private final AnalysisCachePublisher publisher = new AnalysisCachePublisher(analysisCacheEnabled, branchConfiguration, writeCache);
private final AnalysisCachePublisher publisher = new AnalysisCachePublisher(writeCache);

private ScannerReportWriter scannerReportWriter;

@Before
public void before() throws IOException {
scannerReportWriter = new ScannerReportWriter(temp.newFolder());
FileStructure fileStructure = new FileStructure(temp.newFolder());
scannerReportWriter = new ScannerReportWriter(fileStructure);
}

@Test
public void publish_does_nothing_if_cache_not_enabled() {
when(analysisCacheEnabled.isEnabled()).thenReturn(false);
public void publish_closes_cache() {
publisher.publish(scannerReportWriter);
verifyNoInteractions(writeCache);
assertThat(scannerReportWriter.getFileStructure().root()).isEmptyDirectory();
}

@Test
public void publish_does_nothing_if_pull_request() {
when(analysisCacheEnabled.isEnabled()).thenReturn(true);
when(branchConfiguration.isPullRequest()).thenReturn(true);
publisher.publish(scannerReportWriter);
verifyNoInteractions(writeCache);
assertThat(scannerReportWriter.getFileStructure().root()).isEmptyDirectory();
}

@Test
public void publish_cache() {
when(writeCache.getCache()).thenReturn(Map.of("key1", "value1".getBytes(StandardCharsets.UTF_8)));
when(analysisCacheEnabled.isEnabled()).thenReturn(true);
publisher.publish(scannerReportWriter);
verify(writeCache, times(2)).getCache();
assertThat(scannerReportWriter.getFileStructure().analysisCache()).exists();
}

@Test
public void publish_empty_cache() {
when(writeCache.getCache()).thenReturn(emptyMap());
when(analysisCacheEnabled.isEnabled()).thenReturn(true);
publisher.publish(scannerReportWriter);
verify(writeCache).getCache();
assertThat(scannerReportWriter.getFileStructure().analysisCache()).doesNotExist();
verify(writeCache).close();
}
}

+ 12
- 22
sonar-scanner-engine/src/test/java/org/sonar/scanner/report/AnalysisContextReportPublisherTest.java View File

@@ -41,6 +41,7 @@ import org.sonar.core.platform.PluginInfo;
import org.sonar.scanner.bootstrap.GlobalServerSettings;
import org.sonar.scanner.bootstrap.ScannerPluginRepository;
import org.sonar.scanner.fs.InputModuleHierarchy;
import org.sonar.scanner.protocol.output.FileStructure;
import org.sonar.scanner.protocol.output.ScannerReportWriter;
import org.sonar.scanner.scan.ProjectServerSettings;
import org.sonar.scanner.scan.filesystem.InputComponentStore;
@@ -64,30 +65,26 @@ public class AnalysisContextReportPublisherTest {
public TemporaryFolder temp = new TemporaryFolder();

private final ScannerPluginRepository pluginRepo = mock(ScannerPluginRepository.class);
private AnalysisContextReportPublisher publisher;
private System2 system2;
private GlobalServerSettings globalServerSettings;
private InputModuleHierarchy hierarchy;
private InputComponentStore store;
private ProjectServerSettings projectServerSettings;
private final System2 system2 = mock(System2.class);
private final GlobalServerSettings globalServerSettings = mock(GlobalServerSettings.class);
private final InputModuleHierarchy hierarchy = mock(InputModuleHierarchy.class);
private final InputComponentStore store = mock(InputComponentStore.class);
private final ProjectServerSettings projectServerSettings = mock(ProjectServerSettings.class);
private final AnalysisContextReportPublisher publisher = new AnalysisContextReportPublisher(projectServerSettings, pluginRepo, system2, globalServerSettings, hierarchy, store);
private ScannerReportWriter writer;

@Before
public void prepare() {
public void prepare() throws IOException {
logTester.setLevel(LoggerLevel.INFO);
system2 = mock(System2.class);
FileStructure fileStructure = new FileStructure(temp.newFolder());
writer = new ScannerReportWriter(fileStructure);
when(system2.properties()).thenReturn(new Properties());
globalServerSettings = mock(GlobalServerSettings.class);
hierarchy = mock(InputModuleHierarchy.class);
store = mock(InputComponentStore.class);
projectServerSettings = mock(ProjectServerSettings.class);
publisher = new AnalysisContextReportPublisher(projectServerSettings, pluginRepo, system2, globalServerSettings, hierarchy, store);
}

@Test
public void shouldOnlyDumpPluginsByDefault() throws Exception {
when(pluginRepo.getExternalPluginsInfos()).thenReturn(singletonList(new PluginInfo("xoo").setName("Xoo").setVersion(Version.create("1.0"))));

ScannerReportWriter writer = new ScannerReportWriter(temp.newFolder());
DefaultInputModule rootModule = new DefaultInputModule(ProjectDefinition.create()
.setBaseDir(temp.newFolder())
.setWorkDir(temp.newFolder()));
@@ -104,7 +101,6 @@ public class AnalysisContextReportPublisherTest {
@Test
public void dumpServerSideGlobalProps() throws Exception {
logTester.setLevel(LoggerLevel.DEBUG);
ScannerReportWriter writer = new ScannerReportWriter(temp.newFolder());
when(globalServerSettings.properties()).thenReturn(ImmutableMap.of(COM_FOO, "bar", SONAR_SKIP, "true"));
DefaultInputModule rootModule = new DefaultInputModule(ProjectDefinition.create()
.setBaseDir(temp.newFolder())
@@ -123,7 +119,6 @@ public class AnalysisContextReportPublisherTest {
@Test
public void dumpServerSideProjectProps() throws Exception {
logTester.setLevel(LoggerLevel.DEBUG);
ScannerReportWriter writer = new ScannerReportWriter(temp.newFolder());

DefaultInputModule rootModule = new DefaultInputModule(ProjectDefinition.create()
.setBaseDir(temp.newFolder())
@@ -150,7 +145,6 @@ public class AnalysisContextReportPublisherTest {

@Test
public void shouldNotDumpSensitiveModuleProperties() throws Exception {
ScannerReportWriter writer = new ScannerReportWriter(temp.newFolder());
DefaultInputModule rootModule = new DefaultInputModule(ProjectDefinition.create()
.setBaseDir(temp.newFolder())
.setWorkDir(temp.newFolder())
@@ -175,7 +169,6 @@ public class AnalysisContextReportPublisherTest {
@Test
public void shouldShortenModuleProperties() throws Exception {
File baseDir = temp.newFolder();
ScannerReportWriter writer = new ScannerReportWriter(temp.newFolder());
DefaultInputModule rootModule = new DefaultInputModule(ProjectDefinition.create()
.setBaseDir(baseDir)
.setWorkDir(temp.newFolder())
@@ -190,14 +183,13 @@ public class AnalysisContextReportPublisherTest {

assertThat(FileUtils.readFileToString(writer.getFileStructure().analysisLog(), StandardCharsets.UTF_8)).containsSubsequence(
"sonar.aVeryLongProp=" + StringUtils.repeat("abcde", 199) + "ab...",
"sonar.projectBaseDir=" + baseDir.toString(),
"sonar.projectBaseDir=" + baseDir,
"sonar.projectKey=foo");
}

// SONAR-7598
@Test
public void shouldNotDumpSensitiveGlobalProperties() throws Exception {
ScannerReportWriter writer = new ScannerReportWriter(temp.newFolder());
when(globalServerSettings.properties()).thenReturn(ImmutableMap.of("sonar.login", "my_token", "sonar.password", "azerty", "sonar.cpp.license.secured", "AZERTY"));
DefaultInputModule rootModule = new DefaultInputModule(ProjectDefinition.create()
.setBaseDir(temp.newFolder())
@@ -217,7 +209,6 @@ public class AnalysisContextReportPublisherTest {
@Test
public void dontDumpParentProps() throws Exception {
logTester.setLevel(LoggerLevel.DEBUG);
ScannerReportWriter writer = new ScannerReportWriter(temp.newFolder());

DefaultInputModule module = new DefaultInputModule(ProjectDefinition.create()
.setBaseDir(temp.newFolder())
@@ -253,7 +244,6 @@ public class AnalysisContextReportPublisherTest {

@Test
public void init_splitsPluginsByTypeInTheFile() throws IOException {
ScannerReportWriter writer = new ScannerReportWriter(temp.newFolder());
DefaultInputModule parent = new DefaultInputModule(ProjectDefinition.create()
.setBaseDir(temp.newFolder())
.setWorkDir(temp.newFolder())

+ 12
- 11
sonar-scanner-engine/src/test/java/org/sonar/scanner/report/AnalysisWarningsPublisherTest.java View File

@@ -23,12 +23,14 @@ import com.google.common.collect.Lists;
import java.io.File;
import java.io.IOException;
import java.util.List;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.sonar.api.notifications.AnalysisWarnings;
import org.sonar.api.utils.System2;
import org.sonar.scanner.notifications.DefaultAnalysisWarnings;
import org.sonar.scanner.protocol.output.FileStructure;
import org.sonar.scanner.protocol.output.ScannerReport;
import org.sonar.scanner.protocol.output.ScannerReportReader;
import org.sonar.scanner.protocol.output.ScannerReportWriter;
@@ -41,19 +43,18 @@ public class AnalysisWarningsPublisherTest {
@Rule
public TemporaryFolder temp = new TemporaryFolder();

private final AnalysisWarnings analysisWarnings;
private final AnalysisWarningsPublisher underTest;
private final DefaultAnalysisWarnings analysisWarnings = new DefaultAnalysisWarnings(mock(System2.class));
private final AnalysisWarningsPublisher underTest = new AnalysisWarningsPublisher(analysisWarnings);
private FileStructure fileStructure;

public AnalysisWarningsPublisherTest() {
DefaultAnalysisWarnings defaultAnalysisWarnings = new DefaultAnalysisWarnings(mock(System2.class));
this.analysisWarnings = defaultAnalysisWarnings;
this.underTest = new AnalysisWarningsPublisher(defaultAnalysisWarnings);
@Before
public void setUp() throws IOException {
fileStructure = new FileStructure(temp.newFolder());
}

@Test
public void publish_warnings() throws IOException {
File outputDir = temp.newFolder();
ScannerReportWriter writer = new ScannerReportWriter(outputDir);
ScannerReportWriter writer = new ScannerReportWriter(fileStructure);

String warning1 = "warning 1";
String warning2 = "warning 2";
@@ -63,7 +64,7 @@ public class AnalysisWarningsPublisherTest {

underTest.publish(writer);

ScannerReportReader reader = new ScannerReportReader(outputDir);
ScannerReportReader reader = new ScannerReportReader(fileStructure);
List<ScannerReport.AnalysisWarning> warnings = Lists.newArrayList(reader.readAnalysisWarnings());

assertThat(warnings)
@@ -74,13 +75,13 @@ public class AnalysisWarningsPublisherTest {
@Test
public void do_not_write_warnings_report_when_empty() throws IOException {
File outputDir = temp.newFolder();
ScannerReportWriter writer = new ScannerReportWriter(outputDir);
ScannerReportWriter writer = new ScannerReportWriter(fileStructure);

underTest.publish(writer);

assertThat(writer.getFileStructure().analysisWarnings()).doesNotExist();

ScannerReportReader reader = new ScannerReportReader(outputDir);
ScannerReportReader reader = new ScannerReportReader(fileStructure);
List<ScannerReport.AnalysisWarning> warnings = Lists.newArrayList(reader.readAnalysisWarnings());

assertThat(warnings).isEmpty();

+ 5
- 2
sonar-scanner-engine/src/test/java/org/sonar/scanner/report/ChangedLinesPublisherTest.java View File

@@ -37,6 +37,7 @@ import org.sonar.api.batch.fs.internal.TestInputFileBuilder;
import org.sonar.api.batch.scm.ScmProvider;
import org.sonar.api.utils.log.LogTester;
import org.sonar.scanner.fs.InputModuleHierarchy;
import org.sonar.scanner.protocol.output.FileStructure;
import org.sonar.scanner.protocol.output.ScannerReportReader;
import org.sonar.scanner.protocol.output.ScannerReportWriter;
import org.sonar.scanner.repository.ReferenceBranchSupplier;
@@ -65,6 +66,7 @@ public class ChangedLinesPublisherTest {
private final BranchConfiguration branchConfiguration = mock(BranchConfiguration.class);
private final ReferenceBranchSupplier referenceBranchSupplier = mock(ReferenceBranchSupplier.class);
private ScannerReportWriter writer;
private FileStructure fileStructure;
private final ScmProvider provider = mock(ScmProvider.class);
private final DefaultInputProject project = mock(DefaultInputProject.class);

@@ -78,7 +80,8 @@ public class ChangedLinesPublisherTest {

@Before
public void setUp() {
writer = new ScannerReportWriter(temp.getRoot());
fileStructure = new FileStructure(temp.getRoot());
writer = new ScannerReportWriter(fileStructure);
when(branchConfiguration.isPullRequest()).thenReturn(true);
when(scmConfiguration.isDisabled()).thenReturn(false);
when(scmConfiguration.provider()).thenReturn(provider);
@@ -198,7 +201,7 @@ public class ChangedLinesPublisherTest {

private void assertPublished(DefaultInputFile file, Set<Integer> lines) {
assertThat(new File(temp.getRoot(), "changed-lines-" + file.scannerId() + ".pb")).exists();
ScannerReportReader reader = new ScannerReportReader(temp.getRoot());
ScannerReportReader reader = new ScannerReportReader(fileStructure);
assertThat(reader.readComponentChangedLines(file.scannerId()).getLineList()).containsExactlyElementsOf(lines);
}


+ 7
- 6
sonar-scanner-engine/src/test/java/org/sonar/scanner/report/ComponentsPublisherTest.java View File

@@ -56,15 +56,16 @@ public class ComponentsPublisherTest {
@Rule
public TemporaryFolder temp = new TemporaryFolder();

private File outputDir;
private FileStructure fileStructure;
private ScannerReportWriter writer;
private BranchConfiguration branchConfiguration;

@Before
public void setUp() throws IOException {
branchConfiguration = mock(BranchConfiguration.class);
outputDir = temp.newFolder();
writer = new ScannerReportWriter(outputDir);
File outputDir = temp.newFolder();
fileStructure = new FileStructure(outputDir);
writer = new ScannerReportWriter(fileStructure);
}

@Test
@@ -124,7 +125,7 @@ public class ComponentsPublisherTest {
// no such reference
assertThat(writer.hasComponentData(FileStructure.Domain.COMPONENT, 8)).isFalse();

ScannerReportReader reader = new ScannerReportReader(outputDir);
ScannerReportReader reader = new ScannerReportReader(fileStructure);
Component rootProtobuf = reader.readComponent(1);
assertThat(rootProtobuf.getKey()).isEqualTo("foo");
assertThat(rootProtobuf.getDescription()).isEqualTo("Root description");
@@ -197,7 +198,7 @@ public class ComponentsPublisherTest {

assertThat(writer.hasComponentData(FileStructure.Domain.COMPONENT, 1)).isTrue();

ScannerReportReader reader = new ScannerReportReader(outputDir);
ScannerReportReader reader = new ScannerReportReader(fileStructure);
Component rootProtobuf = reader.readComponent(1);
assertThat(rootProtobuf.getKey()).isEqualTo("foo");
assertThat(rootProtobuf.getName()).isEmpty();
@@ -225,7 +226,7 @@ public class ComponentsPublisherTest {
ComponentsPublisher publisher = new ComponentsPublisher(project, store);
publisher.publish(writer);

ScannerReportReader reader = new ScannerReportReader(outputDir);
ScannerReportReader reader = new ScannerReportReader(fileStructure);
Component rootProtobuf = reader.readComponent(1);
assertThat(rootProtobuf.getLinkCount()).isEqualTo(2);
assertThat(rootProtobuf.getLink(0).getType()).isEqualTo(ComponentLinkType.HOME);

+ 23
- 43
sonar-scanner-engine/src/test/java/org/sonar/scanner/report/MetadataPublisherTest.java View File

@@ -23,7 +23,6 @@ import com.google.common.collect.ImmutableMap;
import com.tngtech.java.junit.dataprovider.DataProvider;
import com.tngtech.java.junit.dataprovider.DataProviderRunner;
import com.tngtech.java.junit.dataprovider.UseDataProvider;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -47,6 +46,7 @@ import org.sonar.scanner.bootstrap.ScannerPlugin;
import org.sonar.scanner.bootstrap.ScannerPluginRepository;
import org.sonar.scanner.cpd.CpdSettings;
import org.sonar.scanner.fs.InputModuleHierarchy;
import org.sonar.scanner.protocol.output.FileStructure;
import org.sonar.scanner.protocol.output.ScannerReport;
import org.sonar.scanner.protocol.output.ScannerReportReader;
import org.sonar.scanner.protocol.output.ScannerReportWriter;
@@ -84,9 +84,14 @@ public class MetadataPublisherTest {
private final ScmProvider scmProvider = mock(ScmProvider.class);
private final ScmRevision scmRevision = mock(ScmRevision.class);
private final InputComponentStore componentStore = mock(InputComponentStore.class);
private ScannerReportWriter writer;
private ScannerReportReader reader;

@Before
public void prepare() throws IOException {
FileStructure fileStructure = new FileStructure(temp.newFolder());
writer = new ScannerReportWriter(fileStructure);
reader = new ScannerReportReader(fileStructure);
when(projectInfo.getAnalysisDate()).thenReturn(new Date(1234567L));
when(scmProvider.relativePathFromScmRoot(any(Path.class))).thenReturn(Paths.get("dummy/path"));
when(scmProvider.revisionId(any(Path.class))).thenReturn("dummy-sha1");
@@ -120,18 +125,15 @@ public class MetadataPublisherTest {
}

@Test
public void write_metadata() throws Exception {
public void write_metadata() {
Date date = new Date();
when(qProfiles.findAll()).thenReturn(Collections.singletonList(new QProfile("q1", "Q1", "java", date)));
when(pluginRepository.getPluginsByKey()).thenReturn(ImmutableMap.of(
"java", new ScannerPlugin("java", 12345L, PluginType.BUNDLED, null),
"java", new ScannerPlugin("java", 12345L, PluginType.BUNDLED, null),
"php", new ScannerPlugin("php", 45678L, PluginType.BUNDLED, null)));
File outputDir = temp.newFolder();
ScannerReportWriter writer = new ScannerReportWriter(outputDir);
when(referenceBranchSupplier.getFromProperties()).thenReturn("newCodeReference");
underTest.publish(writer);

ScannerReportReader reader = new ScannerReportReader(outputDir);
ScannerReport.Metadata metadata = reader.readMetadata();
assertThat(metadata.getAnalysisDate()).isEqualTo(1234567L);
assertThat(metadata.getNewCodeReferenceBranch()).isEqualTo("newCodeReference");
@@ -146,9 +148,9 @@ public class MetadataPublisherTest {
.setRulesUpdatedAt(date.getTime())
.build()));
assertThat(metadata.getPluginsByKey()).containsOnly(entry("java", org.sonar.scanner.protocol.output.ScannerReport.Metadata.Plugin.newBuilder()
.setKey("java")
.setUpdatedAt(12345)
.build()),
.setKey("java")
.setUpdatedAt(12345)
.build()),
entry("php", org.sonar.scanner.protocol.output.ScannerReport.Metadata.Plugin.newBuilder()
.setKey("php")
.setUpdatedAt(45678)
@@ -156,30 +158,22 @@ public class MetadataPublisherTest {
}

@Test
public void write_not_analysed_file_counts() throws Exception {
public void write_not_analysed_file_counts() {
when(componentStore.getNotAnalysedFilesByLanguage()).thenReturn(ImmutableMap.of("c", 10, "cpp", 20));

File outputDir = temp.newFolder();
ScannerReportWriter writer = new ScannerReportWriter(outputDir);

underTest.publish(writer);

ScannerReportReader reader = new ScannerReportReader(outputDir);
ScannerReport.Metadata metadata = reader.readMetadata();
assertThat(metadata.getNotAnalyzedFilesByLanguageMap()).contains(entry("c", 10), entry("cpp", 20));
}

@Test
@UseDataProvider("projectVersions")
public void write_project_version(@Nullable String projectVersion, String expected) throws Exception {
public void write_project_version(@Nullable String projectVersion, String expected) {
when(projectInfo.getProjectVersion()).thenReturn(Optional.ofNullable(projectVersion));

File outputDir = temp.newFolder();
ScannerReportWriter writer = new ScannerReportWriter(outputDir);

underTest.publish(writer);

ScannerReportReader reader = new ScannerReportReader(outputDir);
ScannerReport.Metadata metadata = reader.readMetadata();
assertThat(metadata.getProjectVersion()).isEqualTo(expected);
}
@@ -197,15 +191,11 @@ public class MetadataPublisherTest {

@Test
@UseDataProvider("buildStrings")
public void write_buildString(@Nullable String buildString, String expected) throws Exception {
public void write_buildString(@Nullable String buildString, String expected) {
when(projectInfo.getBuildString()).thenReturn(Optional.ofNullable(buildString));

File outputDir = temp.newFolder();
ScannerReportWriter writer = new ScannerReportWriter(outputDir);

underTest.publish(writer);

ScannerReportReader reader = new ScannerReportReader(outputDir);
ScannerReport.Metadata metadata = reader.readMetadata();
assertThat(metadata.getBuildString()).isEqualTo(expected);
}
@@ -222,7 +212,7 @@ public class MetadataPublisherTest {
}

@Test
public void write_branch_info() throws Exception {
public void write_branch_info() {
String branchName = "name";
String targetName = "target";

@@ -230,10 +220,8 @@ public class MetadataPublisherTest {
when(branches.branchType()).thenReturn(BranchType.BRANCH);
when(branches.targetBranchName()).thenReturn(targetName);

File outputDir = temp.newFolder();
underTest.publish(new ScannerReportWriter(outputDir));
underTest.publish(writer);

ScannerReportReader reader = new ScannerReportReader(outputDir);
ScannerReport.Metadata metadata = reader.readMetadata();
assertThat(metadata.getBranchName()).isEqualTo(branchName);
assertThat(metadata.getBranchType()).isEqualTo(ScannerReport.Metadata.BranchType.BRANCH);
@@ -242,29 +230,25 @@ public class MetadataPublisherTest {
}

@Test
public void dont_write_new_code_reference_if_not_specified_in_properties() throws IOException {
public void dont_write_new_code_reference_if_not_specified_in_properties() {
when(referenceBranchSupplier.get()).thenReturn("ref");
when(referenceBranchSupplier.getFromProperties()).thenReturn(null);

File outputDir = temp.newFolder();
underTest.publish(new ScannerReportWriter(outputDir));
underTest.publish(writer);

ScannerReportReader reader = new ScannerReportReader(outputDir);
ScannerReport.Metadata metadata = reader.readMetadata();

assertThat(metadata.getNewCodeReferenceBranch()).isEmpty();
}

@Test
public void write_project_basedir() throws Exception {
public void write_project_basedir() {
String path = "some/dir";
Path relativePathFromScmRoot = Paths.get(path);
when(scmProvider.relativePathFromScmRoot(any(Path.class))).thenReturn(relativePathFromScmRoot);

File outputDir = temp.newFolder();
underTest.publish(new ScannerReportWriter(outputDir));
underTest.publish(writer);

ScannerReportReader reader = new ScannerReportReader(outputDir);
ScannerReport.Metadata metadata = reader.readMetadata();
assertThat(metadata.getRelativePathFromScmRoot()).isEqualTo(path);
}
@@ -274,16 +258,14 @@ public class MetadataPublisherTest {
String revisionId = "some-sha1";
when(scmRevision.get()).thenReturn(Optional.of(revisionId));

File outputDir = temp.newFolder();
underTest.publish(new ScannerReportWriter(outputDir));
underTest.publish(writer);

ScannerReportReader reader = new ScannerReportReader(outputDir);
ScannerReport.Metadata metadata = reader.readMetadata();
assertThat(metadata.getScmRevisionId()).isEqualTo(revisionId);
}

@Test
public void should_not_crash_when_scm_provider_does_not_support_relativePathFromScmRoot() throws IOException {
public void should_not_crash_when_scm_provider_does_not_support_relativePathFromScmRoot() {
ScmProvider fakeScmProvider = new ScmProvider() {
@Override
public String key() {
@@ -292,10 +274,8 @@ public class MetadataPublisherTest {
};
when(scmConfiguration.provider()).thenReturn(fakeScmProvider);

File outputDir = temp.newFolder();
underTest.publish(new ScannerReportWriter(outputDir));
underTest.publish(writer);

ScannerReportReader reader = new ScannerReportReader(outputDir);
ScannerReport.Metadata metadata = reader.readMetadata();
assertThat(metadata.getRelativePathFromScmRoot()).isEmpty();
}

+ 8
- 8
sonar-scanner-engine/src/test/java/org/sonar/scanner/report/ReportPublisherTest.java View File

@@ -42,6 +42,7 @@ import org.sonar.api.utils.log.LoggerLevel;
import org.sonar.scanner.bootstrap.DefaultScannerWsClient;
import org.sonar.scanner.bootstrap.GlobalAnalysisMode;
import org.sonar.scanner.fs.InputModuleHierarchy;
import org.sonar.scanner.protocol.output.FileStructure;
import org.sonar.scanner.protocol.output.ScannerReport;
import org.sonar.scanner.scan.ScanProperties;
import org.sonar.scanner.scan.branch.BranchConfiguration;
@@ -83,6 +84,7 @@ public class ReportPublisherTest {
private CeTaskReportDataHolder reportMetadataHolder = mock(CeTaskReportDataHolder.class);
private ReportPublisher underTest;
private AnalysisWarnings analysisWarnings = mock(AnalysisWarnings.class);
private FileStructure fileStructure;
private JavaArchitectureInformationProvider javaArchitectureInformationProvider = mock(JavaArchitectureInformationProvider.class);

@Before
@@ -90,13 +92,14 @@ public class ReportPublisherTest {
root = new DefaultInputModule(
ProjectDefinition.create().setKey("org.sonarsource.sonarqube:sonarqube").setBaseDir(reportTempFolder.newDir()).setWorkDir(reportTempFolder.getRoot()));
when(moduleHierarchy.root()).thenReturn(root);
fileStructure = new FileStructure(reportTempFolder.getRoot());
when(server.getPublicRootUrl()).thenReturn("https://localhost");
when(server.getVersion()).thenReturn("6.4");
when(properties.metadataFilePath()).thenReturn(reportTempFolder.newDir().toPath()
.resolve("folder")
.resolve("report-task.txt"));
underTest = new ReportPublisher(properties, wsClient, server, contextPublisher, moduleHierarchy, mode, reportTempFolder,
new ReportPublisherStep[0], branchConfiguration, reportMetadataHolder, analysisWarnings, javaArchitectureInformationProvider);
new ReportPublisherStep[0], branchConfiguration, reportMetadataHolder, analysisWarnings, javaArchitectureInformationProvider, fileStructure);
}

@Test
@@ -193,7 +196,7 @@ public class ReportPublisherTest {
when(branchConfiguration.branchType()).thenReturn(BRANCH);
when(branchConfiguration.branchName()).thenReturn("branch-6.7");
ReportPublisher underTest = new ReportPublisher(properties, wsClient, server, contextPublisher, moduleHierarchy, mode, mock(TempFolder.class),
new ReportPublisherStep[0], branchConfiguration, reportMetadataHolder, analysisWarnings, javaArchitectureInformationProvider);
new ReportPublisherStep[0], branchConfiguration, reportMetadataHolder, analysisWarnings, javaArchitectureInformationProvider, fileStructure);

underTest.prepareAndDumpMetadata("TASK-123");

@@ -214,7 +217,7 @@ public class ReportPublisherTest {
when(branchConfiguration.pullRequestKey()).thenReturn("105");

ReportPublisher underTest = new ReportPublisher(properties, wsClient, server, contextPublisher, moduleHierarchy, mode, mock(TempFolder.class),
new ReportPublisherStep[0], branchConfiguration, reportMetadataHolder, analysisWarnings, javaArchitectureInformationProvider);
new ReportPublisherStep[0], branchConfiguration, reportMetadataHolder, analysisWarnings, javaArchitectureInformationProvider, fileStructure);

underTest.prepareAndDumpMetadata("TASK-123");

@@ -279,20 +282,17 @@ public class ReportPublisherTest {
@Test
public void should_not_delete_report_if_property_is_set() throws IOException {
when(properties.shouldKeepReport()).thenReturn(true);
Path reportDir = reportTempFolder.getRoot().toPath().resolve("scanner-report");

underTest.start();
underTest.stop();
assertThat(reportDir).isDirectory();
assertThat(fileStructure.root()).isDirectory();
}

@Test
public void should_delete_report_by_default() throws IOException {
Path reportDir = reportTempFolder.getRoot().toPath().resolve("scanner-report");

underTest.start();
underTest.stop();
assertThat(reportDir).doesNotExist();
assertThat(fileStructure.root()).doesNotExist();
}

@Test

+ 3
- 1
sonar-scanner-engine/src/test/java/org/sonar/scanner/report/SourcePublisherTest.java View File

@@ -31,6 +31,7 @@ import org.sonar.api.SonarRuntime;
import org.sonar.api.batch.fs.internal.DefaultInputFile;
import org.sonar.api.batch.fs.internal.DefaultInputProject;
import org.sonar.api.batch.fs.internal.TestInputFileBuilder;
import org.sonar.scanner.protocol.output.FileStructure;
import org.sonar.scanner.protocol.output.ScannerReportWriter;
import org.sonar.scanner.scan.branch.BranchConfiguration;
import org.sonar.scanner.scan.filesystem.InputComponentStore;
@@ -64,7 +65,8 @@ public class SourcePublisherTest {

publisher = new SourcePublisher(componentStore);
File outputDir = temp.newFolder();
writer = new ScannerReportWriter(outputDir);
FileStructure fileStructure = new FileStructure(outputDir);
writer = new ScannerReportWriter(fileStructure);
}

@Test

+ 3
- 2
sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/DefaultSensorStorageTest.java View File

@@ -96,8 +96,9 @@ public class DefaultSensorStorageTest {

reportPublisher = mock(ReportPublisher.class);
final File reportDir = temp.newFolder();
reportWriter = new ScannerReportWriter(reportDir);
reportReader = new ScannerReportReader(reportDir);
FileStructure fileStructure = new FileStructure(reportDir);
reportWriter = new ScannerReportWriter(fileStructure);
reportReader = new ScannerReportReader(fileStructure);
when(reportPublisher.getWriter()).thenReturn(reportWriter);
when(reportPublisher.getReader()).thenReturn(reportReader);


+ 39
- 0
sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/internal/SensorCacheData.java View File

@@ -0,0 +1,39 @@
/*
* SonarQube
* Copyright (C) 2009-2023 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.scanner.protocol.internal;

import com.google.protobuf.ByteString;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.sonar.scanner.protocol.internal.ScannerInternal.SensorCacheEntry;

public class SensorCacheData {
private final Map<String, ByteString> map;

public SensorCacheData(List<SensorCacheEntry> entries) {
this.map = Collections.unmodifiableMap(entries.stream().collect(Collectors.toMap(SensorCacheEntry::getKey, SensorCacheEntry::getData)));
}

public Map<String, ByteString> getEntries() {
return map;
}
}

+ 24
- 0
sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/internal/package-info.java View File

@@ -0,0 +1,24 @@
/*
* SonarQube
* Copyright (C) 2009-2023 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
@ParametersAreNonnullByDefault
package org.sonar.scanner.protocol.internal;

import javax.annotation.ParametersAreNonnullByDefault;


+ 1
- 1
sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/FileStructure.java View File

@@ -67,7 +67,7 @@ public class FileStructure {
}

public File analysisCache() {
return new File(dir, "analysis-cache.pb");
return new File(dir, "analysis-cache2.pb");
}

public File analysisLog() {

+ 2
- 3
sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportReader.java View File

@@ -31,11 +31,10 @@ import org.sonar.core.util.Protobuf;
import static org.sonar.core.util.CloseableIterator.emptyCloseableIterator;

public class ScannerReportReader {

private final FileStructure fileStructure;

public ScannerReportReader(File dir) {
this.fileStructure = new FileStructure(dir);
public ScannerReportReader(FileStructure fileStructure) {
this.fileStructure = fileStructure;
}

public ScannerReport.Metadata readMetadata() {

+ 2
- 12
sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportWriter.java View File

@@ -30,14 +30,10 @@ import org.sonar.scanner.protocol.internal.ScannerInternal;

@Immutable
public class ScannerReportWriter {

private final FileStructure fileStructure;

public ScannerReportWriter(File dir) {
if (!dir.exists() && !dir.mkdirs()) {
throw new IllegalStateException("Unable to create directory: " + dir);
}
this.fileStructure = new FileStructure(dir);
public ScannerReportWriter(FileStructure fileStructure) {
this.fileStructure = fileStructure;
}

public FileStructure getFileStructure() {
@@ -95,12 +91,6 @@ public class ScannerReportWriter {
return file;
}

public File writeAnalysisCache(ScannerInternal.AnalysisCacheMsg cacheMsg) {
File file = fileStructure.analysisCache();
Protobuf.writeGzip(cacheMsg, file);
return file;
}

public void appendComponentExternalIssue(int componentRef, ScannerReport.ExternalIssue issue) {
File file = fileStructure.fileFor(FileStructure.Domain.EXTERNAL_ISSUES, componentRef);
try (OutputStream out = new BufferedOutputStream(new FileOutputStream(file, true))) {

+ 25
- 22
sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/viewer/ScannerReportViewerApp.java View File

@@ -19,7 +19,10 @@
*/
package org.sonar.scanner.protocol.viewer;

import java.awt.*;
import java.awt.BorderLayout;
import java.awt.Cursor;
import java.awt.Dimension;
import java.awt.EventQueue;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
@@ -31,7 +34,16 @@ import java.util.List;
import java.util.Map;
import java.util.Scanner;
import javax.annotation.CheckForNull;
import javax.swing.*;
import javax.swing.JEditorPane;
import javax.swing.JFileChooser;
import javax.swing.JFrame;
import javax.swing.JOptionPane;
import javax.swing.JPanel;
import javax.swing.JScrollPane;
import javax.swing.JSplitPane;
import javax.swing.JTabbedPane;
import javax.swing.JTree;
import javax.swing.UIManager;
import javax.swing.UIManager.LookAndFeelInfo;
import javax.swing.event.TreeSelectionEvent;
import javax.swing.event.TreeSelectionListener;
@@ -41,6 +53,7 @@ import javax.swing.tree.TreeSelectionModel;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.sonar.core.util.CloseableIterator;
import org.sonar.scanner.protocol.output.FileStructure;
import org.sonar.scanner.protocol.output.FileStructure.Domain;
import org.sonar.scanner.protocol.output.ScannerReport;
import org.sonar.scanner.protocol.output.ScannerReport.Changesets;
@@ -111,19 +124,15 @@ public class ScannerReportViewerApp {
* Launch the application.
*/
public static void main(String[] args) {
EventQueue.invokeLater(new Runnable() {
@Override
public void run() {
try {
ScannerReportViewerApp window = new ScannerReportViewerApp();
window.frame.setVisible(true);

window.loadReport();
} catch (Exception e) {
e.printStackTrace();
}
}
EventQueue.invokeLater(() -> {
try {
ScannerReportViewerApp window = new ScannerReportViewerApp();
window.frame.setVisible(true);

window.loadReport();
} catch (Exception e) {
e.printStackTrace();
}
});
}

@@ -184,7 +193,8 @@ public class ScannerReportViewerApp {
}

private void loadReport(File file) {
reader = new ScannerReportReader(file);
FileStructure fileStructure = new FileStructure(file);
reader = new ScannerReportReader(fileStructure);
metadata = reader.readMetadata();
updateTitle();
loadComponents();
@@ -651,13 +661,6 @@ public class ScannerReportViewerApp {
return componentTree;
}

/**
* @wbp.factory
*/
public static JPanel createComponentPanel() {
return new JPanel();
}

protected JEditorPane getComponentEditor() {
return componentEditor;
}

+ 3
- 2
sonar-scanner-protocol/src/main/protobuf/scanner_internal.proto View File

@@ -3,6 +3,7 @@ syntax = "proto3";
option java_package = "org.sonar.scanner.protocol.internal";
option optimize_for = SPEED;

message AnalysisCacheMsg {
map<string, bytes> map = 1;
message SensorCacheEntry {
string key = 1;
bytes data = 2;
}

+ 41
- 0
sonar-scanner-protocol/src/test/java/org/sonar/scanner/protocol/internal/SensorCacheDataTest.java View File

@@ -0,0 +1,41 @@
/*
* SonarQube
* Copyright (C) 2009-2023 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.scanner.protocol.internal;

import com.google.protobuf.ByteString;
import java.util.List;
import org.junit.Test;
import org.sonar.scanner.protocol.internal.ScannerInternal.SensorCacheEntry;

import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Map.entry;
import static org.assertj.core.api.Assertions.assertThat;

public class SensorCacheDataTest {
@Test
public void constructor_processes_entries() {
SensorCacheEntry entry1 = SensorCacheEntry.newBuilder().setKey("key1").setData(ByteString.copyFrom("data1", UTF_8)).build();
SensorCacheEntry entry2 = SensorCacheEntry.newBuilder().setKey("key2").setData(ByteString.copyFrom("data2", UTF_8)).build();

SensorCacheData data = new SensorCacheData(List.of(entry1, entry2));
assertThat(data.getEntries()).containsExactly(entry(entry1.getKey(), entry1.getData()), entry(entry2.getKey(), entry2.getData()));
}

}

+ 45
- 37
sonar-scanner-protocol/src/test/java/org/sonar/scanner/protocol/output/ScannerReportReaderTest.java View File

@@ -25,8 +25,8 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.AbstractMap;
import java.util.zip.GZIPInputStream;
import java.util.LinkedList;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.junit.Before;
@@ -35,8 +35,7 @@ import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.sonar.core.util.CloseableIterator;
import org.sonar.core.util.Protobuf;
import org.sonar.scanner.protocol.internal.ScannerInternal;
import org.sonar.scanner.protocol.internal.ScannerInternal.AnalysisCacheMsg;
import org.sonar.scanner.protocol.internal.ScannerInternal.SensorCacheEntry;
import org.sonar.scanner.protocol.output.ScannerReport.Measure.StringValue;
import org.sonar.scanner.protocol.output.ScannerReport.SyntaxHighlightingRule.HighlightingType;

@@ -47,24 +46,24 @@ import static org.assertj.core.api.Assertions.assertThat;

public class ScannerReportReaderTest {

private static int UNKNOWN_COMPONENT_REF = 123;
private static final int UNKNOWN_COMPONENT_REF = 123;

@Rule
public TemporaryFolder temp = new TemporaryFolder();

private File dir;

private FileStructure fileStructure;
private ScannerReportReader underTest;

@Before
public void setUp() throws Exception {
dir = temp.newFolder();
underTest = new ScannerReportReader(dir);
File dir = temp.newFolder();
fileStructure = new FileStructure(dir);
underTest = new ScannerReportReader(fileStructure);
}

@Test
public void read_metadata() {
ScannerReportWriter writer = new ScannerReportWriter(dir);
ScannerReportWriter writer = new ScannerReportWriter(fileStructure);
ScannerReport.Metadata.Builder metadata = ScannerReport.Metadata.newBuilder()
.setAnalysisDate(15000000L)
.setProjectKey("PROJECT_A")
@@ -86,7 +85,7 @@ public class ScannerReportReaderTest {

@Test
public void read_components() {
ScannerReportWriter writer = new ScannerReportWriter(dir);
ScannerReportWriter writer = new ScannerReportWriter(fileStructure);
ScannerReport.Component.Builder component = ScannerReport.Component.newBuilder()
.setRef(1)
.setProjectRelativePath("src/main/java/Foo.java");
@@ -102,7 +101,7 @@ public class ScannerReportReaderTest {

@Test
public void read_issues() {
ScannerReportWriter writer = new ScannerReportWriter(dir);
ScannerReportWriter writer = new ScannerReportWriter(fileStructure);
ScannerReport.Issue issue = ScannerReport.Issue.newBuilder()
.build();
writer.writeComponentIssues(1, asList(issue));
@@ -113,7 +112,7 @@ public class ScannerReportReaderTest {

@Test
public void read_external_issues() {
ScannerReportWriter writer = new ScannerReportWriter(dir);
ScannerReportWriter writer = new ScannerReportWriter(fileStructure);
ScannerReport.ExternalIssue issue = ScannerReport.ExternalIssue.newBuilder()
.build();
writer.appendComponentExternalIssue(1, issue);
@@ -129,7 +128,7 @@ public class ScannerReportReaderTest {

@Test
public void read_measures() {
ScannerReportWriter writer = new ScannerReportWriter(dir);
ScannerReportWriter writer = new ScannerReportWriter(fileStructure);
ScannerReport.Measure.Builder measure = ScannerReport.Measure.newBuilder()
.setStringValue(StringValue.newBuilder().setValue("value_a"));
writer.appendComponentMeasure(1, measure.build());
@@ -144,7 +143,7 @@ public class ScannerReportReaderTest {

@Test
public void read_changesets() {
ScannerReportWriter writer = new ScannerReportWriter(dir);
ScannerReportWriter writer = new ScannerReportWriter(fileStructure);
ScannerReport.Changesets.Builder scm = ScannerReport.Changesets.newBuilder()
.setComponentRef(1)
.addChangeset(ScannerReport.Changesets.Changeset.newBuilder().setDate(123_456_789).setAuthor("jack.daniels").setRevision("123-456-789"));
@@ -161,7 +160,7 @@ public class ScannerReportReaderTest {

@Test
public void read_duplications() {
ScannerReportWriter writer = new ScannerReportWriter(dir);
ScannerReportWriter writer = new ScannerReportWriter(fileStructure);
writer.writeMetadata(ScannerReport.Metadata.newBuilder()
.setRootComponentRef(1).build());
writer.writeComponent(ScannerReport.Component.newBuilder()
@@ -182,7 +181,7 @@ public class ScannerReportReaderTest {
.build();
writer.writeComponentDuplications(1, asList(duplication));

ScannerReportReader sut = new ScannerReportReader(dir);
ScannerReportReader sut = new ScannerReportReader(fileStructure);
assertThat(sut.readComponentDuplications(1)).toIterable().hasSize(1);
}

@@ -193,7 +192,7 @@ public class ScannerReportReaderTest {

@Test
public void read_duplication_blocks() {
ScannerReportWriter writer = new ScannerReportWriter(dir);
ScannerReportWriter writer = new ScannerReportWriter(fileStructure);
writer.writeMetadata(ScannerReport.Metadata.newBuilder()
.setRootComponentRef(1).build());
writer.writeComponent(ScannerReport.Component.newBuilder()
@@ -208,37 +207,46 @@ public class ScannerReportReaderTest {
.build();
writer.writeCpdTextBlocks(1, singletonList(duplicationBlock));

ScannerReportReader sut = new ScannerReportReader(dir);
ScannerReportReader sut = new ScannerReportReader(fileStructure);
assertThat(sut.readCpdTextBlocks(1)).toIterable().hasSize(1);
}

@Test
public void read_analysis_cache() throws IOException {
ScannerReportWriter writer = new ScannerReportWriter(dir);
writer.writeAnalysisCache(ScannerInternal.AnalysisCacheMsg.newBuilder()
.putMap("key", ByteString.copyFrom("data", UTF_8))
.build());
ScannerReportWriter writer = new ScannerReportWriter(fileStructure);

SensorCacheEntry entry1 = SensorCacheEntry.newBuilder()
.setKey("key")
.setData(ByteString.copyFrom("data", UTF_8))
.build();
SensorCacheEntry entry2 = SensorCacheEntry.newBuilder()
.setKey("key")
.setData(ByteString.copyFrom("data", UTF_8))
.build();

ScannerReportReader reader = new ScannerReportReader(dir);
Protobuf.writeStream(List.of(entry1, entry2), fileStructure.analysisCache(), false);
ScannerReportReader reader = new ScannerReportReader(fileStructure);

AnalysisCacheMsg cache = Protobuf.read(new GZIPInputStream(reader.getAnalysisCache()), ScannerInternal.AnalysisCacheMsg.parser());
assertThat(cache.getMapMap()).containsOnly(new AbstractMap.SimpleEntry<>("key", ByteString.copyFrom("data", UTF_8)));
CloseableIterator<SensorCacheEntry> it = Protobuf.readStream(reader.getAnalysisCache(), SensorCacheEntry.parser());
List<SensorCacheEntry> data = new LinkedList<>();
it.forEachRemaining(data::add);
assertThat(data).containsExactly(entry1, entry2);
}

@Test
public void read_analysis_cache_returns_null_if_no_file_exists() {
ScannerReportReader reader = new ScannerReportReader(dir);
ScannerReportReader reader = new ScannerReportReader(fileStructure);
assertThat(reader.getAnalysisCache()).isNull();
}

@Test
@Test
public void empty_list_if_no_duplication_block_found() {
assertThat(underTest.readComponentDuplications(UNKNOWN_COMPONENT_REF)).toIterable().isEmpty();
}

@Test
public void read_syntax_highlighting() {
ScannerReportWriter writer = new ScannerReportWriter(dir);
ScannerReportWriter writer = new ScannerReportWriter(fileStructure);
writer.writeMetadata(ScannerReport.Metadata.newBuilder()
.setRootComponentRef(1)
.build());
@@ -270,7 +278,7 @@ public class ScannerReportReaderTest {

@Test
public void read_symbols() {
ScannerReportWriter writer = new ScannerReportWriter(dir);
ScannerReportWriter writer = new ScannerReportWriter(fileStructure);
writer.writeMetadata(ScannerReport.Metadata.newBuilder()
.setRootComponentRef(1)
.build());
@@ -292,7 +300,7 @@ public class ScannerReportReaderTest {
.build())
.build()));

underTest = new ScannerReportReader(dir);
underTest = new ScannerReportReader(fileStructure);
assertThat(underTest.readComponentSymbols(1)).toIterable().hasSize(1);
}

@@ -303,7 +311,7 @@ public class ScannerReportReaderTest {

@Test
public void read_coverage() {
ScannerReportWriter writer = new ScannerReportWriter(dir);
ScannerReportWriter writer = new ScannerReportWriter(fileStructure);
writer.writeMetadata(ScannerReport.Metadata.newBuilder()
.setRootComponentRef(1)
.build());
@@ -324,8 +332,8 @@ public class ScannerReportReaderTest {
.setCoveredConditions(4)
.build()));

underTest = new ScannerReportReader(dir);
try (CloseableIterator<ScannerReport.LineCoverage> it = new ScannerReportReader(dir).readComponentCoverage(1)) {
underTest = new ScannerReportReader(fileStructure);
try (CloseableIterator<ScannerReport.LineCoverage> it = new ScannerReportReader(fileStructure).readComponentCoverage(1)) {
ScannerReport.LineCoverage coverage = it.next();
assertThat(coverage.getLine()).isOne();
assertThat(coverage.getConditions()).isOne();
@@ -341,17 +349,17 @@ public class ScannerReportReaderTest {

@Test
public void read_source_lines() throws Exception {
ScannerReportWriter writer = new ScannerReportWriter(dir);
ScannerReportWriter writer = new ScannerReportWriter(fileStructure);
File file = writer.getFileStructure().fileFor(FileStructure.Domain.SOURCE, 1);
FileUtils.writeLines(file, Lists.newArrayList("line1", "line2"));

File sourceFile = new ScannerReportReader(dir).readFileSource(1);
File sourceFile = new ScannerReportReader(fileStructure).readFileSource(1);
assertThat(sourceFile).isEqualTo(file);
}

@Test
public void read_file_source() throws Exception {
ScannerReportWriter writer = new ScannerReportWriter(dir);
ScannerReportWriter writer = new ScannerReportWriter(fileStructure);
try (FileOutputStream outputStream = new FileOutputStream(writer.getSourceFile(1))) {
IOUtils.write("line1\nline2", outputStream);
}

+ 4
- 11
sonar-scanner-protocol/src/test/java/org/sonar/scanner/protocol/output/ScannerReportWriterTest.java View File

@@ -41,20 +41,13 @@ public class ScannerReportWriterTest {
@Rule
public TemporaryFolder temp = new TemporaryFolder();
File dir;
ScannerReportWriter underTest;
private FileStructure fileStructure;
private ScannerReportWriter underTest;

@Before
public void setUp() throws Exception {
dir = temp.newFolder();
underTest = new ScannerReportWriter(dir);
}

@Test
public void create_dir_if_does_not_exist() {
FileUtils.deleteQuietly(dir);
underTest = new ScannerReportWriter(dir);

assertThat(dir).isDirectory().exists();
fileStructure = new FileStructure(temp.newFolder());
underTest = new ScannerReportWriter(fileStructure);
}

@Test

Loading…
Cancel
Save