Browse Source

SONAR-16097 Add plugin cache to the Sensor API

tags/9.4.0.54424
Duarte Meneses 2 years ago
parent
commit
ecf844b394
31 changed files with 1161 additions and 36 deletions
  1. 34
    0
      sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/sensor/internal/SensorContextTester.java
  2. 21
    0
      sonar-plugin-api-impl/src/test/java/org/sonar/api/batch/sensor/internal/SensorContextTesterTest.java
  3. 40
    0
      sonar-plugin-api/src/main/java/org/sonar/api/Beta.java
  4. 32
    0
      sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/SensorContext.java
  5. 38
    0
      sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/cache/ReadCache.java
  6. 44
    0
      sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/cache/WriteCache.java
  7. 35
    0
      sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheEnabled.java
  8. 87
    0
      sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheLoader.java
  9. 59
    0
      sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheMemoryStorage.java
  10. 83
    0
      sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheProvider.java
  11. 30
    0
      sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheStorage.java
  12. 47
    0
      sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/ReadCacheImpl.java
  13. 27
    0
      sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/ScannerWriteCache.java
  14. 81
    0
      sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/WriteCacheImpl.java
  15. 11
    10
      sonar-scanner-engine/src/main/java/org/sonar/scanner/report/AnalysisCachePublisher.java
  16. 14
    2
      sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java
  17. 6
    2
      sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java
  18. 26
    1
      sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java
  19. 45
    0
      sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheEnabledTest.java
  20. 97
    0
      sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheLoaderTest.java
  21. 66
    0
      sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheMemoryStorageTest.java
  22. 65
    0
      sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheProviderTest.java
  23. 52
    0
      sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/ReadCacheImplTest.java
  24. 85
    0
      sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/WriteCacheImplTest.java
  25. 9
    9
      sonar-scanner-engine/src/test/java/org/sonar/scanner/report/AnalysisCachePublisherTest.java
  26. 16
    2
      sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ModuleSensorContextTest.java
  27. 1
    1
      sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/FileStructure.java
  28. 3
    3
      sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportReader.java
  29. 2
    2
      sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportWriter.java
  30. 1
    1
      sonar-scanner-protocol/src/main/protobuf/scanner_internal.proto
  31. 4
    3
      sonar-scanner-protocol/src/test/java/org/sonar/scanner/protocol/output/ScannerReportReaderTest.java

+ 34
- 0
sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/sensor/internal/SensorContextTester.java View File

@@ -47,6 +47,8 @@ import org.sonar.api.batch.rule.ActiveRules;
import org.sonar.api.batch.rule.internal.ActiveRulesBuilder;
import org.sonar.api.batch.sensor.Sensor;
import org.sonar.api.batch.sensor.SensorContext;
import org.sonar.api.batch.sensor.cache.ReadCache;
import org.sonar.api.batch.sensor.cache.WriteCache;
import org.sonar.api.batch.sensor.code.NewSignificantCode;
import org.sonar.api.batch.sensor.code.internal.DefaultSignificantCode;
import org.sonar.api.batch.sensor.coverage.NewCoverage;
@@ -110,8 +112,11 @@ public class SensorContextTester implements SensorContext {
private DefaultInputProject project;
private DefaultInputModule module;
private SonarRuntime runtime;
private ReadCache readCache;
private WriteCache writeCache;
private boolean canSkipUnchangedFiles;
private boolean cancelled;
private boolean cacheEnabled = false;

private SensorContextTester(Path moduleBaseDir) {
this.settings = new MapSettings();
@@ -408,6 +413,35 @@ public class SensorContextTester implements SensorContext {
file.setPublished(true);
}

@Override
public WriteCache nextCache() {
return writeCache;
}

public void setNextCache(WriteCache writeCache) {
this.writeCache = writeCache;
}

@Override
public ReadCache previousAnalysisCache() {
return readCache;
}


public void setPreviousAnalysisCache(ReadCache cache) {
this.readCache = cache;
}

@Override
public boolean isCacheEnabled() {
return cacheEnabled;
}


public void setCacheEnabled(boolean enabled) {
this.cacheEnabled = enabled;
}

@Override
public NewSignificantCode newSignificantCode() {
return new DefaultSignificantCode(sensorStorage);

+ 21
- 0
sonar-plugin-api-impl/src/test/java/org/sonar/api/batch/sensor/internal/SensorContextTesterTest.java View File

@@ -36,6 +36,8 @@ import org.sonar.api.batch.rule.ActiveRules;
import org.sonar.api.batch.rule.Severity;
import org.sonar.api.batch.rule.internal.ActiveRulesBuilder;
import org.sonar.api.batch.rule.internal.NewActiveRule;
import org.sonar.api.batch.sensor.cache.ReadCache;
import org.sonar.api.batch.sensor.cache.WriteCache;
import org.sonar.api.batch.sensor.error.AnalysisError;
import org.sonar.api.batch.sensor.error.NewAnalysisError;
import org.sonar.api.batch.sensor.highlighting.TypeOfText;
@@ -51,6 +53,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.assertj.core.api.Assertions.tuple;
import static org.assertj.core.data.MapEntry.entry;
import static org.mockito.Mockito.mock;

public class SensorContextTesterTest {

@@ -81,6 +84,24 @@ public class SensorContextTesterTest {
assertThat(tester.canSkipUnchangedFiles()).isTrue();
}

@Test
public void testPluginCache() {
assertThat(tester.nextCache()).isNull();
assertThat(tester.previousAnalysisCache()).isNull();
assertThat(tester.isCacheEnabled()).isFalse();

ReadCache readCache = mock(ReadCache.class);
WriteCache writeCache = mock(WriteCache.class);

tester.setPreviousAnalysisCache(readCache);
tester.setNextCache(writeCache);
tester.setCacheEnabled(true);

assertThat(tester.nextCache()).isEqualTo(writeCache);
assertThat(tester.previousAnalysisCache()).isEqualTo(readCache);
assertThat(tester.isCacheEnabled()).isTrue();
}

@Test
public void testActiveRules() {
NewActiveRule activeRule = new NewActiveRule.Builder()

+ 40
- 0
sonar-plugin-api/src/main/java/org/sonar/api/Beta.java View File

@@ -0,0 +1,40 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.api;

import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;

/**
* This annotations marks API as experimental.
* API marked with it can be subject to breaking changes or removal in any future version without compliance with deprecation policies.
*/
@Retention(RetentionPolicy.CLASS)
@Target({
ElementType.ANNOTATION_TYPE,
ElementType.CONSTRUCTOR,
ElementType.FIELD,
ElementType.METHOD,
ElementType.TYPE})
@Documented
public @interface Beta {}

+ 32
- 0
sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/SensorContext.java View File

@@ -20,11 +20,14 @@
package org.sonar.api.batch.sensor;

import java.io.Serializable;
import org.sonar.api.Beta;
import org.sonar.api.SonarRuntime;
import org.sonar.api.batch.fs.FileSystem;
import org.sonar.api.batch.fs.InputFile;
import org.sonar.api.batch.fs.InputModule;
import org.sonar.api.batch.rule.ActiveRules;
import org.sonar.api.batch.sensor.cache.ReadCache;
import org.sonar.api.batch.sensor.cache.WriteCache;
import org.sonar.api.batch.sensor.code.NewSignificantCode;
import org.sonar.api.batch.sensor.coverage.NewCoverage;
import org.sonar.api.batch.sensor.cpd.NewCpdTokens;
@@ -218,4 +221,33 @@ public interface SensorContext {
*/
void markForPublishing(InputFile inputFile);

/**
* Access object to write cache that will be stored and made available in a future analysis.
* If cache is disabled, the methods in the returned object will have no effect.
* This API is experimental and can be changed or dropped at any time.
* @see #isCacheEnabled()
* @since 9.4
*/
@Beta
WriteCache nextCache();

/**
* Access object to read cached data. The origin of the cached data is not specified and could come from a different branch.
* If cache is disabled, the methods in the returned object will have no effect.
* This API is experimental and can be changed or dropped at any time.
* @see #isCacheEnabled()
* @since 9.4
*/
@Beta
ReadCache previousAnalysisCache();

/**
* Returns true if caching is enabled.
* This API is experimental and can be changed or dropped at any time.
* @see #nextCache()
* @see #previousAnalysisCache()
* @since 9.4
*/
@Beta
boolean isCacheEnabled();
}

+ 38
- 0
sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/cache/ReadCache.java View File

@@ -0,0 +1,38 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.api.batch.sensor.cache;

import java.io.InputStream;
import org.sonar.api.Beta;

@Beta
public interface ReadCache {
/**
* Returns an input stream for the data cached with the key.
* It's the responsibility of the caller to close the stream.
* @throws IllegalArgumentException if cache doesn't contain key
*/
InputStream read(String key);

/**
* Checks whether the cache contains a key
*/
boolean contains(String key);
}

+ 44
- 0
sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/cache/WriteCache.java View File

@@ -0,0 +1,44 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.api.batch.sensor.cache;

import java.io.InputStream;
import org.sonar.api.Beta;

@Beta
public interface WriteCache {
/**
* Save a new entry in the cache. The stream will be consumed immediately.
* @throws IllegalArgumentException if the cache already contains the key
*/
void write(String key, InputStream data);

/**
* Save a new entry in the cache.
* @throws IllegalArgumentException if the cache already contains the key
*/
void write(String key, byte[] data);

/**
* Copy a cached entry from the previous cache to the new one.
* @throws IllegalArgumentException if the previous cache doesn't contain given key or if this cache already contains the key
*/
void copyFromPrevious(String key);
}

+ 35
- 0
sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheEnabled.java View File

@@ -0,0 +1,35 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.scanner.cache;

import org.sonar.api.config.Configuration;

public class AnalysisCacheEnabled {
static final String PROP_KEY = "sonar.analysisCache.enabled";
private final Configuration configuration;

public AnalysisCacheEnabled(Configuration configuration) {
this.configuration = configuration;
}

public boolean isEnabled() {
return configuration.getBoolean(PROP_KEY).orElse(false);
}
}

+ 87
- 0
sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheLoader.java View File

@@ -0,0 +1,87 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.scanner.cache;

import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.util.Optional;
import java.util.zip.InflaterInputStream;
import org.sonar.api.scanner.fs.InputProject;
import org.sonar.core.util.Protobuf;
import org.sonar.scanner.bootstrap.DefaultScannerWsClient;
import org.sonar.scanner.protocol.internal.ScannerInternal;
import org.sonar.scanner.protocol.internal.ScannerInternal.AnalysisCacheMsg;
import org.sonar.scanner.scan.branch.BranchConfiguration;
import org.sonar.scanner.scan.branch.BranchType;
import org.sonarqube.ws.client.GetRequest;
import org.sonarqube.ws.client.WsResponse;

/**
* Loads plugin cache into the local storage
*/
public class AnalysisCacheLoader {
static final String CONTENT_ENCODING = "Content-Encoding";
static final String ACCEPT_ENCODING = "Accept-Encoding";
private static final String URL = "api/scanner_cache/get";

private final DefaultScannerWsClient wsClient;
private final InputProject project;
private final BranchConfiguration branchConfiguration;

public AnalysisCacheLoader(DefaultScannerWsClient wsClient, InputProject project, BranchConfiguration branchConfiguration) {
this.project = project;
this.branchConfiguration = branchConfiguration;
this.wsClient = wsClient;
}

public Optional<AnalysisCacheMsg> load() {
String url = URL + "?project=" + project.key();
if (branchConfiguration.branchType() == BranchType.BRANCH && branchConfiguration.branchName() != null) {
url = url + "&branch=" + branchConfiguration.branchName();
}

GetRequest request = new GetRequest(url).setHeader(ACCEPT_ENCODING, "gzip");

try (WsResponse response = wsClient.call(request)) {
if (response.code() == HttpURLConnection.HTTP_NOT_FOUND) {
return Optional.empty();
}
try (InputStream is = response.contentStream()) {
Optional<String> contentEncoding = response.header(CONTENT_ENCODING);
if (contentEncoding.isPresent() && contentEncoding.get().equals("gzip")) {
return Optional.of(decompress(is));
} else {
return Optional.of(Protobuf.read(is, AnalysisCacheMsg.parser()));
}
} catch (IOException e) {
throw new IllegalStateException("Failed to download cache", e);
}
}
}

private static AnalysisCacheMsg decompress(InputStream is) {
try (InflaterInputStream iis = new InflaterInputStream(is)) {
return Protobuf.read(iis, ScannerInternal.AnalysisCacheMsg.parser());
} catch (IOException e) {
throw new IllegalStateException("Failed to decompress plugin cache", e);
}
}
}

+ 59
- 0
sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheMemoryStorage.java View File

@@ -0,0 +1,59 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.scanner.cache;

import java.io.InputStream;
import javax.annotation.CheckForNull;
import javax.annotation.Nullable;
import org.sonar.scanner.protocol.internal.ScannerInternal.AnalysisCacheMsg;

public class AnalysisCacheMemoryStorage implements AnalysisCacheStorage {
private final AnalysisCacheLoader loader;
@Nullable
private AnalysisCacheMsg cache;

public AnalysisCacheMemoryStorage(AnalysisCacheLoader loader) {
this.loader = loader;
}

@Override
@CheckForNull
public InputStream get(String key) {
if (cache == null) {
return null;
}
if (cache.containsMap(key)) {
return cache.getMapOrThrow(key).newInput();
}
return null;
}

@Override
public boolean contains(String key) {
if (cache == null) {
return false;
}
return cache.containsMap(key);
}

public void load() {
cache = loader.load().orElse(null);
}
}

+ 83
- 0
sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheProvider.java View File

@@ -0,0 +1,83 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.scanner.cache;

import java.io.InputStream;
import java.util.Map;
import org.jetbrains.annotations.Nullable;
import org.sonar.api.batch.sensor.cache.ReadCache;
import org.springframework.context.annotation.Bean;

import static java.util.Collections.emptyMap;

public class AnalysisCacheProvider {
@Bean("ReadCache")
public ReadCache provideReader(AnalysisCacheEnabled analysisCacheEnabled, AnalysisCacheMemoryStorage storage) {
if (analysisCacheEnabled.isEnabled()) {
storage.load();
return new ReadCacheImpl(storage);
}
return new NoOpReadCache();
}

@Bean("WriteCache")
public ScannerWriteCache provideWriter(AnalysisCacheEnabled analysisCacheEnabled, ReadCache readCache) {
if (analysisCacheEnabled.isEnabled()) {
return new WriteCacheImpl(readCache);
}
return new NoOpWriteCache();
}


static class NoOpWriteCache implements ScannerWriteCache {
@Override
public void write(String s, InputStream inputStream) {
// no op
}

@Override
public void write(String s, byte[] bytes) {
// no op
}

@Override
public void copyFromPrevious(String s) {
// no op
}

@Override
public Map<String, byte[]> getCache() {
return emptyMap();
}
}

static class NoOpReadCache implements ReadCache {
@Nullable
@Override
public InputStream read(String s) {
return null;
}

@Override
public boolean contains(String s) {
return false;
}
}
}

+ 30
- 0
sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheStorage.java View File

@@ -0,0 +1,30 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.scanner.cache;

import java.io.InputStream;
import javax.annotation.CheckForNull;

public interface AnalysisCacheStorage {
@CheckForNull
InputStream get(String key);

boolean contains(String key);
}

+ 47
- 0
sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/ReadCacheImpl.java View File

@@ -0,0 +1,47 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.scanner.cache;

import java.io.InputStream;
import org.sonar.api.batch.sensor.cache.ReadCache;

import static org.sonar.api.utils.Preconditions.checkArgument;
import static org.sonar.api.utils.Preconditions.checkNotNull;

public class ReadCacheImpl implements ReadCache {
private final AnalysisCacheStorage cache;

public ReadCacheImpl(AnalysisCacheStorage storage) {
this.cache = storage;
}

@Override
public InputStream read(String key) {
checkNotNull(key);
checkArgument(contains(key));
return cache.get(key);
}

@Override
public boolean contains(String key) {
checkNotNull(key);
return cache.contains(key);
}
}

+ 27
- 0
sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/ScannerWriteCache.java View File

@@ -0,0 +1,27 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.scanner.cache;

import java.util.Map;
import org.sonar.api.batch.sensor.cache.WriteCache;

public interface ScannerWriteCache extends WriteCache {
Map<String, byte[]> getCache();
}

+ 81
- 0
sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/WriteCacheImpl.java View File

@@ -0,0 +1,81 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.scanner.cache;

import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import org.sonar.api.batch.sensor.cache.ReadCache;

import static java.util.Collections.unmodifiableMap;
import static org.sonar.api.utils.Preconditions.checkArgument;
import static org.sonar.api.utils.Preconditions.checkNotNull;

public class WriteCacheImpl implements ScannerWriteCache {
private final ReadCache readCache;
private final Map<String, byte[]> cache = new HashMap<>();

public WriteCacheImpl(ReadCache readCache) {
this.readCache = readCache;
}

@Override
public void write(String key, InputStream data) {
checkNotNull(data);
checkKey(key);
try {
byte[] arr = data.readAllBytes();
cache.put(key, arr);
} catch (IOException e) {
throw new IllegalStateException("Failed to read stream", e);
}
}

@Override
public void write(String key, byte[] data) {
checkNotNull(data);
checkKey(key);
cache.put(key, Arrays.copyOf(data, data.length));
}

@Override
public void copyFromPrevious(String key) {
checkArgument(readCache.contains(key), "Previous cache doesn't contain key '%s'", key);
checkKey(key);

try {
cache.put(key, readCache.read(key).readAllBytes());
} catch (IOException e) {
throw new IllegalStateException("Failed to read plugin cache for key " + key, e);
}
}

@Override
public Map<String, byte[]> getCache() {
return unmodifiableMap(cache);
}

private void checkKey(String key) {
checkNotNull(key);
checkArgument(!cache.containsKey(key), "Cache already contains key '%s'", key);
}
}

sonar-scanner-engine/src/main/java/org/sonar/scanner/report/PluginCachePublisher.java → sonar-scanner-engine/src/main/java/org/sonar/scanner/report/AnalysisCachePublisher.java View File

@@ -21,31 +21,32 @@ package org.sonar.scanner.report;

import com.google.protobuf.ByteString;
import java.util.Map;
import org.sonar.scanner.cache.PluginCacheEnabled;
import org.sonar.scanner.cache.AnalysisCacheEnabled;
import org.sonar.scanner.cache.ScannerWriteCache;
import org.sonar.scanner.protocol.internal.ScannerInternal.PluginCacheMsg;
import org.sonar.scanner.protocol.internal.ScannerInternal;
import org.sonar.scanner.protocol.internal.ScannerInternal.AnalysisCacheMsg;
import org.sonar.scanner.protocol.output.ScannerReportWriter;

public class PluginCachePublisher implements ReportPublisherStep {
private final PluginCacheEnabled pluginCacheEnabled;
public class AnalysisCachePublisher implements ReportPublisherStep {
private final AnalysisCacheEnabled analysisCacheEnabled;
private final ScannerWriteCache cache;

public PluginCachePublisher(PluginCacheEnabled pluginCacheEnabled, ScannerWriteCache cache) {
this.pluginCacheEnabled = pluginCacheEnabled;
public AnalysisCachePublisher(AnalysisCacheEnabled analysisCacheEnabled, ScannerWriteCache cache) {
this.analysisCacheEnabled = analysisCacheEnabled;
this.cache = cache;
}

@Override
public void publish(ScannerReportWriter writer) {
if (!pluginCacheEnabled.isEnabled() || cache.getCache().isEmpty()) {
if (!analysisCacheEnabled.isEnabled() || cache.getCache().isEmpty()) {
return;
}
PluginCacheMsg.Builder pluginCacheMsg = PluginCacheMsg.newBuilder();
AnalysisCacheMsg.Builder analysisCacheMsg = ScannerInternal.AnalysisCacheMsg.newBuilder();

for (Map.Entry<String, byte[]> entry : cache.getCache().entrySet()) {
pluginCacheMsg.putMap(entry.getKey(), ByteString.copyFrom(entry.getValue()));
analysisCacheMsg.putMap(entry.getKey(), ByteString.copyFrom(entry.getValue()));
}

writer.writePluginCache(pluginCacheMsg.build());
writer.writeAnalysisCache(analysisCacheMsg.build());
}
}

+ 14
- 2
sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java View File

@@ -35,6 +35,7 @@ import org.sonar.core.config.ScannerProperties;
import org.sonar.core.extension.CoreExtensionsInstaller;
import org.sonar.core.language.LanguagesProvider;
import org.sonar.core.metric.ScannerMetrics;
import org.sonar.core.platform.SpringComponentContainer;
import org.sonar.scanner.DefaultFileLinesContextFactory;
import org.sonar.scanner.ProjectInfo;
import org.sonar.scanner.analysis.AnalysisTempFolderProvider;
@@ -42,7 +43,10 @@ import org.sonar.scanner.bootstrap.ExtensionInstaller;
import org.sonar.scanner.bootstrap.ExtensionMatcher;
import org.sonar.scanner.bootstrap.GlobalAnalysisMode;
import org.sonar.scanner.bootstrap.PostJobExtensionDictionary;
import org.sonar.core.platform.SpringComponentContainer;
import org.sonar.scanner.cache.AnalysisCacheEnabled;
import org.sonar.scanner.cache.AnalysisCacheLoader;
import org.sonar.scanner.cache.AnalysisCacheMemoryStorage;
import org.sonar.scanner.cache.AnalysisCacheProvider;
import org.sonar.scanner.ci.CiConfigurationProvider;
import org.sonar.scanner.ci.vendors.AppVeyor;
import org.sonar.scanner.ci.vendors.AwsCodeBuild;
@@ -84,6 +88,7 @@ import org.sonar.scanner.report.ChangedLinesPublisher;
import org.sonar.scanner.report.ComponentsPublisher;
import org.sonar.scanner.report.ContextPropertiesPublisher;
import org.sonar.scanner.report.MetadataPublisher;
import org.sonar.scanner.report.AnalysisCachePublisher;
import org.sonar.scanner.report.ReportPublisher;
import org.sonar.scanner.report.SourcePublisher;
import org.sonar.scanner.report.TestExecutionPublisher;
@@ -128,8 +133,8 @@ import org.sonar.scanner.sensor.ProjectSensorsExecutor;
import org.sonar.scm.git.GitScmSupport;
import org.sonar.scm.svn.SvnScmSupport;

import static org.sonar.api.utils.Preconditions.checkNotNull;
import static org.sonar.api.batch.InstantiationStrategy.PER_BATCH;
import static org.sonar.api.utils.Preconditions.checkNotNull;
import static org.sonar.core.extension.CoreExtensionsInstaller.noExtensionFilter;
import static org.sonar.scanner.bootstrap.ExtensionUtils.isDeprecatedScannerSide;
import static org.sonar.scanner.bootstrap.ExtensionUtils.isInstantiationStrategy;
@@ -166,6 +171,7 @@ public class SpringProjectScanContainer extends SpringComponentContainer {
new ProjectPullRequestsProvider(),
ProjectRepositoriesSupplier.class,
new ProjectServerSettingsProvider(),
AnalysisCacheEnabled.class,

// temp
new AnalysisTempFolderProvider(),
@@ -223,6 +229,11 @@ public class SpringProjectScanContainer extends SpringComponentContainer {

ProjectCoverageAndDuplicationExclusions.class,

// Plugin cache
AnalysisCacheProvider.class,
AnalysisCacheMemoryStorage.class,
AnalysisCacheLoader.class,

// Report
ReferenceBranchSupplier.class,
ScannerMetrics.class,
@@ -232,6 +243,7 @@ public class SpringProjectScanContainer extends SpringComponentContainer {
ActiveRulesPublisher.class,
AnalysisWarningsPublisher.class,
ComponentsPublisher.class,
AnalysisCachePublisher.class,
TestExecutionPublisher.class,
SourcePublisher.class,
ChangedLinesPublisher.class,

+ 6
- 2
sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java View File

@@ -25,9 +25,12 @@ import org.sonar.api.batch.fs.FileSystem;
import org.sonar.api.batch.fs.InputModule;
import org.sonar.api.batch.fs.internal.DefaultInputProject;
import org.sonar.api.batch.rule.ActiveRules;
import org.sonar.api.batch.sensor.cache.ReadCache;
import org.sonar.api.batch.sensor.cache.WriteCache;
import org.sonar.api.batch.sensor.internal.SensorStorage;
import org.sonar.api.config.Configuration;
import org.sonar.api.config.Settings;
import org.sonar.scanner.cache.AnalysisCacheEnabled;
import org.sonar.scanner.scan.branch.BranchConfiguration;

@ThreadSafe
@@ -36,8 +39,9 @@ public class ModuleSensorContext extends ProjectSensorContext {
private final InputModule module;

public ModuleSensorContext(DefaultInputProject project, InputModule module, Configuration config, Settings mutableModuleSettings, FileSystem fs, ActiveRules activeRules,
SensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration) {
super(project, config, mutableModuleSettings, fs, activeRules, sensorStorage, sonarRuntime, branchConfiguration);
SensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration,
WriteCache writeCache, ReadCache readCache, AnalysisCacheEnabled analysisCacheEnabled) {
super(project, config, mutableModuleSettings, fs, activeRules, sensorStorage, sonarRuntime, branchConfiguration, writeCache, readCache, analysisCacheEnabled);
this.module = module;
}


+ 26
- 1
sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java View File

@@ -29,6 +29,8 @@ import org.sonar.api.batch.fs.internal.DefaultInputFile;
import org.sonar.api.batch.fs.internal.DefaultInputProject;
import org.sonar.api.batch.rule.ActiveRules;
import org.sonar.api.batch.sensor.SensorContext;
import org.sonar.api.batch.sensor.cache.ReadCache;
import org.sonar.api.batch.sensor.cache.WriteCache;
import org.sonar.api.batch.sensor.code.NewSignificantCode;
import org.sonar.api.batch.sensor.code.internal.DefaultSignificantCode;
import org.sonar.api.batch.sensor.coverage.NewCoverage;
@@ -53,6 +55,7 @@ import org.sonar.api.config.Configuration;
import org.sonar.api.config.Settings;
import org.sonar.api.scanner.fs.InputProject;
import org.sonar.api.utils.Version;
import org.sonar.scanner.cache.AnalysisCacheEnabled;
import org.sonar.scanner.scan.branch.BranchConfiguration;
import org.sonar.scanner.sensor.noop.NoOpNewAnalysisError;

@@ -69,9 +72,13 @@ public class ProjectSensorContext implements SensorContext {
private final SonarRuntime sonarRuntime;
private final Configuration config;
private final boolean skipUnchangedFiles;
private final WriteCache writeCache;
private final ReadCache readCache;
private final AnalysisCacheEnabled analysisCacheEnabled;

public ProjectSensorContext(DefaultInputProject project, Configuration config, Settings mutableSettings, FileSystem fs, ActiveRules activeRules,
SensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration) {
SensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration, WriteCache writeCache, ReadCache readCache,
AnalysisCacheEnabled analysisCacheEnabled) {
this.project = project;
this.config = config;
this.mutableSettings = mutableSettings;
@@ -79,6 +86,9 @@ public class ProjectSensorContext implements SensorContext {
this.activeRules = activeRules;
this.sensorStorage = sensorStorage;
this.sonarRuntime = sonarRuntime;
this.writeCache = writeCache;
this.readCache = readCache;
this.analysisCacheEnabled = analysisCacheEnabled;
this.skipUnchangedFiles = branchConfiguration.isPullRequest();
}

@@ -183,6 +193,21 @@ public class ProjectSensorContext implements SensorContext {
file.setPublished(true);
}

@Override
public WriteCache nextCache() {
return writeCache;
}

@Override
public ReadCache previousAnalysisCache() {
return readCache;
}

@Override
public boolean isCacheEnabled() {
return analysisCacheEnabled.isEnabled();
}

@Override
public NewSignificantCode newSignificantCode() {
return new DefaultSignificantCode(sensorStorage);

+ 45
- 0
sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheEnabledTest.java View File

@@ -0,0 +1,45 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.scanner.cache;

import java.util.Optional;
import org.junit.Test;
import org.sonar.api.config.Configuration;

import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.sonar.scanner.cache.AnalysisCacheEnabled.PROP_KEY;

public class AnalysisCacheEnabledTest {
private final Configuration configuration = mock(Configuration.class);
private final AnalysisCacheEnabled analysisCacheEnabled = new AnalysisCacheEnabled(configuration);

@Test
public void disabled_unless_property_set() {
assertThat(analysisCacheEnabled.isEnabled()).isFalse();
}

@Test
public void enabled_if_property_set() {
when(configuration.getBoolean(PROP_KEY)).thenReturn(Optional.of(true));
assertThat(analysisCacheEnabled.isEnabled()).isTrue();
}
}

+ 97
- 0
sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheLoaderTest.java View File

@@ -0,0 +1,97 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.scanner.cache;

import com.google.protobuf.ByteString;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Optional;
import java.util.zip.DeflaterInputStream;
import org.junit.Before;
import org.junit.Test;
import org.sonar.api.scanner.fs.InputProject;
import org.sonar.scanner.bootstrap.DefaultScannerWsClient;
import org.sonar.scanner.protocol.internal.ScannerInternal;
import org.sonar.scanner.protocol.internal.ScannerInternal.AnalysisCacheMsg;
import org.sonar.scanner.scan.branch.BranchConfiguration;
import org.sonarqube.ws.client.WsResponse;

import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.sonar.scanner.cache.AnalysisCacheLoader.CONTENT_ENCODING;

public class AnalysisCacheLoaderTest {
private final WsResponse response = mock(WsResponse.class);
private final DefaultScannerWsClient wsClient = mock(DefaultScannerWsClient.class);
private final InputProject project = mock(InputProject.class);
private final BranchConfiguration branchConfiguration = mock(BranchConfiguration.class);
private final AnalysisCacheLoader loader = new AnalysisCacheLoader(wsClient, project, branchConfiguration);

@Before
public void before() {
when(wsClient.call(any())).thenReturn(response);
}

@Test
public void loads_content() throws IOException {
ScannerInternal.AnalysisCacheMsg expected = ScannerInternal.AnalysisCacheMsg.newBuilder()
.putMap("key", ByteString.copyFrom("value", StandardCharsets.UTF_8))
.build();
setResponse(expected);
AnalysisCacheMsg msg = loader.load().get();
assertThat(msg).isEqualTo(expected);
}

@Test
public void loads_compressed_content() throws IOException {
AnalysisCacheMsg expected = AnalysisCacheMsg.newBuilder()
.putMap("key", ByteString.copyFrom("value", StandardCharsets.UTF_8))
.build();
setCompressedResponse(expected);
AnalysisCacheMsg msg = loader.load().get();
assertThat(msg).isEqualTo(expected);
}

@Test
public void returns_empty_if_404() {
when(response.code()).thenReturn(404);
assertThat(loader.load()).isEmpty();
}

private void setResponse(AnalysisCacheMsg msg) throws IOException {
when(response.contentStream()).thenReturn(createInputStream(msg));
}

private void setCompressedResponse(AnalysisCacheMsg msg) throws IOException {
when(response.contentStream()).thenReturn(new DeflaterInputStream(createInputStream(msg)));
when(response.header(CONTENT_ENCODING)).thenReturn(Optional.of("gzip"));
}

private InputStream createInputStream(AnalysisCacheMsg analysisCacheMsg) throws IOException {
ByteArrayOutputStream serialized = new ByteArrayOutputStream(analysisCacheMsg.getSerializedSize());
analysisCacheMsg.writeTo(serialized);
return new ByteArrayInputStream(serialized.toByteArray());
}
}

+ 66
- 0
sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheMemoryStorageTest.java View File

@@ -0,0 +1,66 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.scanner.cache;

import com.google.protobuf.ByteString;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Optional;
import org.apache.commons.io.IOUtils;
import org.junit.Test;
import org.sonar.scanner.protocol.internal.ScannerInternal.AnalysisCacheMsg;

import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

public class AnalysisCacheMemoryStorageTest {
private final AnalysisCacheLoader loader = mock(AnalysisCacheLoader.class);
private final AnalysisCacheMemoryStorage storage = new AnalysisCacheMemoryStorage(loader);

@Test
public void storage_loads_with_loader() throws IOException {
when(loader.load()).thenReturn(Optional.of(AnalysisCacheMsg.newBuilder()
.putMap("key1", ByteString.copyFrom("value1", StandardCharsets.UTF_8))
.build()));

storage.load();
verify(loader).load();
assertThat(IOUtils.toString(storage.get("key1"), StandardCharsets.UTF_8)).isEqualTo("value1");
assertThat(storage.contains("key1")).isTrue();
}

@Test
public void get_returns_null_if_doesnt_contain_key() {
when(loader.load()).thenReturn(Optional.of(AnalysisCacheMsg.newBuilder().build()));
storage.load();
assertThat(storage.contains("key1")).isFalse();
assertThat(storage.get("key1")).isNull();
}

@Test
public void get_returns_null_if_no_cache() {
when(loader.load()).thenReturn(Optional.empty());
storage.load();
assertThat(storage.contains("key1")).isFalse();
assertThat(storage.get("key1")).isNull();
}
}

+ 65
- 0
sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheProviderTest.java View File

@@ -0,0 +1,65 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.scanner.cache;

import org.junit.Test;
import org.sonar.api.batch.sensor.cache.ReadCache;

import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

public class AnalysisCacheProviderTest {

AnalysisCacheEnabled analysisCacheEnabled = mock(AnalysisCacheEnabled.class);
AnalysisCacheMemoryStorage storage = mock(AnalysisCacheMemoryStorage.class);
ReadCache readCache = mock(ReadCache.class);
AnalysisCacheProvider cacheProvider = new AnalysisCacheProvider();

@Test
public void provide_noop_reader_cache_when_disable() {
when(analysisCacheEnabled.isEnabled()).thenReturn(false);
var cache = cacheProvider.provideReader(analysisCacheEnabled, storage);
assertThat(cache).isInstanceOf(AnalysisCacheProvider.NoOpReadCache.class);
}

@Test
public void provide_noop_writer_cache_when_disable() {
when(analysisCacheEnabled.isEnabled()).thenReturn(false);
var cache = cacheProvider.provideWriter(analysisCacheEnabled, readCache);
assertThat(cache).isInstanceOf(AnalysisCacheProvider.NoOpWriteCache.class);
}

@Test
public void provide_real_reader_cache_when_enable() {
when(analysisCacheEnabled.isEnabled()).thenReturn(true);
var cache = cacheProvider.provideReader(analysisCacheEnabled, storage);
verify(storage).load();
assertThat(cache).isInstanceOf(ReadCacheImpl.class);
}

@Test
public void provide_real_writer_cache_when_enable() {
when(analysisCacheEnabled.isEnabled()).thenReturn(true);
var cache = cacheProvider.provideWriter(analysisCacheEnabled, readCache);
assertThat(cache).isInstanceOf(WriteCacheImpl.class);
}
}

+ 52
- 0
sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/ReadCacheImplTest.java View File

@@ -0,0 +1,52 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.scanner.cache;

import java.io.InputStream;
import org.junit.Test;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

public class ReadCacheImplTest {
private final AnalysisCacheStorage storage = mock(AnalysisCacheStorage.class);
private final ReadCacheImpl readCache = new ReadCacheImpl(storage);

@Test
public void read_delegates_to_storage() {
InputStream is = mock(InputStream.class);
when(storage.get("key")).thenReturn(is);
when(storage.contains("key")).thenReturn(true);
assertThat(readCache.read("key")).isEqualTo(is);
}

@Test
public void read_fails_if_key_not_found() {
assertThatThrownBy(() -> readCache.read("unknown")).isInstanceOf(IllegalArgumentException.class);
}

@Test
public void contains_delegates_to_storage() {
when(storage.contains("key")).thenReturn(true);
assertThat(readCache.contains("key")).isTrue();
}
}

+ 85
- 0
sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/WriteCacheImplTest.java View File

@@ -0,0 +1,85 @@
/*
* SonarQube
* Copyright (C) 2009-2022 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.scanner.cache;

import java.io.ByteArrayInputStream;
import java.io.InputStream;
import org.junit.Test;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.assertj.core.data.MapEntry.entry;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

public class WriteCacheImplTest {
private final ReadCacheImpl readCache = mock(ReadCacheImpl.class);
private final WriteCacheImpl writeCache = new WriteCacheImpl(readCache);

@Test
public void write_bytes_adds_entries() {
byte[] b1 = new byte[] {1, 2, 3};
byte[] b2 = new byte[] {3, 4};
writeCache.write("key", b1);
writeCache.write("key2", b2);

assertThat(writeCache.getCache()).containsOnly(entry("key", b1), entry("key2", b2));
}

@Test
public void write_inputStream_adds_entries() {
byte[] b1 = new byte[] {1, 2, 3};
byte[] b2 = new byte[] {3, 4};
writeCache.write("key", new ByteArrayInputStream(b1));
writeCache.write("key2", new ByteArrayInputStream(b2));

assertThat(writeCache.getCache()).containsOnly(entry("key", b1), entry("key2", b2));
}

@Test
public void write_throws_IAE_if_writing_same_key_twice() {
byte[] b1 = new byte[] {1};
byte[] b2 = new byte[] {2};


writeCache.write("key", b1);
assertThatThrownBy(() -> writeCache.write("key", b2))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Cache already contains key 'key'");
}

@Test
public void copyFromPrevious_throws_IAE_if_read_cache_doesnt_contain_key() {
assertThatThrownBy(() -> writeCache.copyFromPrevious("key"))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Previous cache doesn't contain key 'key'");
}

@Test
public void copyFromPrevious_reads_from_readCache() {
byte[] b = new byte[] {1};
InputStream value = new ByteArrayInputStream(b);
when(readCache.contains("key")).thenReturn(true);
when(readCache.read("key")).thenReturn(value);
writeCache.copyFromPrevious("key");

assertThat(writeCache.getCache()).containsOnly(entry("key", b));
}
}

sonar-scanner-engine/src/test/java/org/sonar/scanner/report/PluginCachePublisherTest.java → sonar-scanner-engine/src/test/java/org/sonar/scanner/report/AnalysisCachePublisherTest.java View File

@@ -26,7 +26,7 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.sonar.scanner.cache.PluginCacheEnabled;
import org.sonar.scanner.cache.AnalysisCacheEnabled;
import org.sonar.scanner.cache.ScannerWriteCache;
import org.sonar.scanner.protocol.output.ScannerReportWriter;

@@ -38,13 +38,13 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;

public class PluginCachePublisherTest {
public class AnalysisCachePublisherTest {
@Rule
public TemporaryFolder temp = new TemporaryFolder();

private final ScannerWriteCache writeCache = mock(ScannerWriteCache.class);
private final PluginCacheEnabled pluginCacheEnabled = mock(PluginCacheEnabled.class);
private final PluginCachePublisher publisher = new PluginCachePublisher(pluginCacheEnabled, writeCache);
private final AnalysisCacheEnabled analysisCacheEnabled = mock(AnalysisCacheEnabled.class);
private final AnalysisCachePublisher publisher = new AnalysisCachePublisher(analysisCacheEnabled, writeCache);

private ScannerReportWriter scannerReportWriter;

@@ -55,7 +55,7 @@ public class PluginCachePublisherTest {

@Test
public void publish_does_nothing_if_cache_not_enabled() {
when(pluginCacheEnabled.isEnabled()).thenReturn(false);
when(analysisCacheEnabled.isEnabled()).thenReturn(false);
publisher.publish(scannerReportWriter);
verifyNoInteractions(writeCache);
assertThat(scannerReportWriter.getFileStructure().root()).isEmptyDirectory();
@@ -64,18 +64,18 @@ public class PluginCachePublisherTest {
@Test
public void publish_cache() {
when(writeCache.getCache()).thenReturn(Map.of("key1", "value1".getBytes(StandardCharsets.UTF_8)));
when(pluginCacheEnabled.isEnabled()).thenReturn(true);
when(analysisCacheEnabled.isEnabled()).thenReturn(true);
publisher.publish(scannerReportWriter);
verify(writeCache, times(2)).getCache();
assertThat(scannerReportWriter.getFileStructure().pluginCache()).exists();
assertThat(scannerReportWriter.getFileStructure().analysisCache()).exists();
}

@Test
public void publish_empty_cache() {
when(writeCache.getCache()).thenReturn(emptyMap());
when(pluginCacheEnabled.isEnabled()).thenReturn(true);
when(analysisCacheEnabled.isEnabled()).thenReturn(true);
publisher.publish(scannerReportWriter);
verify(writeCache).getCache();
assertThat(scannerReportWriter.getFileStructure().pluginCache()).doesNotExist();
assertThat(scannerReportWriter.getFileStructure().analysisCache()).doesNotExist();
}
}

+ 16
- 2
sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ModuleSensorContextTest.java View File

@@ -37,6 +37,9 @@ import org.sonar.api.config.internal.MapSettings;
import org.sonar.api.internal.SonarRuntimeImpl;
import org.sonar.api.measures.CoreMetrics;
import org.sonar.api.utils.Version;
import org.sonar.scanner.cache.AnalysisCacheEnabled;
import org.sonar.scanner.cache.ReadCacheImpl;
import org.sonar.scanner.cache.WriteCacheImpl;
import org.sonar.scanner.scan.branch.BranchConfiguration;

import static org.assertj.core.api.Assertions.assertThat;
@@ -55,6 +58,9 @@ public class ModuleSensorContextTest {
private SensorStorage sensorStorage;
private SonarRuntime runtime;
private BranchConfiguration branchConfiguration;
private WriteCacheImpl writeCache;
private ReadCacheImpl readCache;
private AnalysisCacheEnabled analysisCacheEnabled;

@Before
public void prepare() throws Exception {
@@ -66,18 +72,25 @@ public class ModuleSensorContextTest {
settings = new MapSettings();
sensorStorage = mock(SensorStorage.class);
branchConfiguration = mock(BranchConfiguration.class);
writeCache = mock(WriteCacheImpl.class);
readCache = mock(ReadCacheImpl.class);
analysisCacheEnabled = mock(AnalysisCacheEnabled.class);
runtime = SonarRuntimeImpl.forSonarQube(Version.parse("5.5"), SonarQubeSide.SCANNER, SonarEdition.COMMUNITY);
}

@Test
public void shouldProvideComponents() {
adaptor = new ModuleSensorContext(mock(DefaultInputProject.class), mock(InputModule.class), settings.asConfig(), settings, fs, activeRules, sensorStorage, runtime, branchConfiguration);
adaptor = new ModuleSensorContext(mock(DefaultInputProject.class), mock(InputModule.class), settings.asConfig(), settings, fs, activeRules, sensorStorage, runtime,
branchConfiguration, writeCache, readCache, analysisCacheEnabled);
assertThat(adaptor.activeRules()).isEqualTo(activeRules);
assertThat(adaptor.fileSystem()).isEqualTo(fs);
assertThat(adaptor.getSonarQubeVersion()).isEqualTo(Version.parse("5.5"));
assertThat(adaptor.runtime()).isEqualTo(runtime);
assertThat(adaptor.canSkipUnchangedFiles()).isFalse();

assertThat(adaptor.nextCache()).isEqualTo(writeCache);
assertThat(adaptor.previousAnalysisCache()).isEqualTo(readCache);

assertThat(adaptor.newIssue()).isNotNull();
assertThat(adaptor.newExternalIssue()).isNotNull();
assertThat(adaptor.newAdHocRule()).isNotNull();
@@ -90,7 +103,8 @@ public class ModuleSensorContextTest {
@Test
public void pull_request_can_skip_unchanged_files() {
when(branchConfiguration.isPullRequest()).thenReturn(true);
adaptor = new ModuleSensorContext(mock(DefaultInputProject.class), mock(InputModule.class), settings.asConfig(), settings, fs, activeRules, sensorStorage, runtime, branchConfiguration);
adaptor = new ModuleSensorContext(mock(DefaultInputProject.class), mock(InputModule.class), settings.asConfig(), settings, fs, activeRules, sensorStorage, runtime,
branchConfiguration, writeCache, readCache, analysisCacheEnabled);
assertThat(adaptor.canSkipUnchangedFiles()).isTrue();
}


+ 1
- 1
sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/FileStructure.java View File

@@ -66,7 +66,7 @@ public class FileStructure {
return new File(dir, "metadata.pb");
}

public File pluginCache() {
public File analysisCache() {
return new File(dir, "plugin-cache.pb");
}


+ 3
- 3
sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportReader.java View File

@@ -81,12 +81,12 @@ public class ScannerReportReader {

@CheckForNull
public InputStream getPluginCache() {
File file = fileStructure.pluginCache();
File file = fileStructure.analysisCache();
if (fileExists(file)) {
try {
return new BufferedInputStream(new FileInputStream(fileStructure.pluginCache()));
return new BufferedInputStream(new FileInputStream(fileStructure.analysisCache()));
} catch (FileNotFoundException e) {
throw new IllegalStateException("Unable to open file " + fileStructure.pluginCache(), e);
throw new IllegalStateException("Unable to open file " + fileStructure.analysisCache(), e);
}
}
return null;

+ 2
- 2
sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportWriter.java View File

@@ -95,8 +95,8 @@ public class ScannerReportWriter {
return file;
}

public File writePluginCache(ScannerInternal.PluginCacheMsg cacheMsg) {
File file = fileStructure.pluginCache();
public File writeAnalysisCache(ScannerInternal.AnalysisCacheMsg cacheMsg) {
File file = fileStructure.analysisCache();
Protobuf.writeGzip(cacheMsg, file);
return file;
}

+ 1
- 1
sonar-scanner-protocol/src/main/protobuf/scanner_internal.proto View File

@@ -3,6 +3,6 @@ syntax = "proto3";
option java_package = "org.sonar.scanner.protocol.internal";
option optimize_for = SPEED;

message PluginCacheMsg {
message AnalysisCacheMsg {
map<string, bytes> map = 1;
}

+ 4
- 3
sonar-scanner-protocol/src/test/java/org/sonar/scanner/protocol/output/ScannerReportReaderTest.java View File

@@ -35,7 +35,8 @@ import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.sonar.core.util.CloseableIterator;
import org.sonar.core.util.Protobuf;
import org.sonar.scanner.protocol.internal.ScannerInternal.PluginCacheMsg;
import org.sonar.scanner.protocol.internal.ScannerInternal;
import org.sonar.scanner.protocol.internal.ScannerInternal.AnalysisCacheMsg;
import org.sonar.scanner.protocol.output.ScannerReport.Measure.StringValue;
import org.sonar.scanner.protocol.output.ScannerReport.SyntaxHighlightingRule.HighlightingType;

@@ -214,13 +215,13 @@ public class ScannerReportReaderTest {
@Test
public void read_plugin_cache() throws IOException {
ScannerReportWriter writer = new ScannerReportWriter(dir);
writer.writePluginCache(PluginCacheMsg.newBuilder()
writer.writeAnalysisCache(ScannerInternal.AnalysisCacheMsg.newBuilder()
.putMap("key", ByteString.copyFrom("data", UTF_8))
.build());

ScannerReportReader reader = new ScannerReportReader(dir);

PluginCacheMsg cache = Protobuf.read(new GZIPInputStream(reader.getPluginCache()), PluginCacheMsg.parser());
AnalysisCacheMsg cache = Protobuf.read(new GZIPInputStream(reader.getPluginCache()), ScannerInternal.AnalysisCacheMsg.parser());
assertThat(cache.getMapMap()).containsOnly(new AbstractMap.SimpleEntry<>("key", ByteString.copyFrom("data", UTF_8)));
}


Loading…
Cancel
Save