diff options
author | Duarte Meneses <duarte.meneses@sonarsource.com> | 2022-03-14 11:28:22 -0500 |
---|---|---|
committer | sonartech <sonartech@sonarsource.com> | 2022-03-18 20:02:57 +0000 |
commit | ecf844b39473fe21acaa1f832d244a4cc4c9f811 (patch) | |
tree | a03e87cb743b877be05c92774e6f009bad3932f5 | |
parent | 1269984e8e09338c057d068d715ade7df5a0c354 (diff) | |
download | sonarqube-ecf844b39473fe21acaa1f832d244a4cc4c9f811.tar.gz sonarqube-ecf844b39473fe21acaa1f832d244a4cc4c9f811.zip |
SONAR-16097 Add plugin cache to the Sensor API
31 files changed, 1161 insertions, 36 deletions
diff --git a/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/sensor/internal/SensorContextTester.java b/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/sensor/internal/SensorContextTester.java index d5d0c27f8da..9cbe24f4eeb 100644 --- a/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/sensor/internal/SensorContextTester.java +++ b/sonar-plugin-api-impl/src/main/java/org/sonar/api/batch/sensor/internal/SensorContextTester.java @@ -47,6 +47,8 @@ import org.sonar.api.batch.rule.ActiveRules; import org.sonar.api.batch.rule.internal.ActiveRulesBuilder; import org.sonar.api.batch.sensor.Sensor; import org.sonar.api.batch.sensor.SensorContext; +import org.sonar.api.batch.sensor.cache.ReadCache; +import org.sonar.api.batch.sensor.cache.WriteCache; import org.sonar.api.batch.sensor.code.NewSignificantCode; import org.sonar.api.batch.sensor.code.internal.DefaultSignificantCode; import org.sonar.api.batch.sensor.coverage.NewCoverage; @@ -110,8 +112,11 @@ public class SensorContextTester implements SensorContext { private DefaultInputProject project; private DefaultInputModule module; private SonarRuntime runtime; + private ReadCache readCache; + private WriteCache writeCache; private boolean canSkipUnchangedFiles; private boolean cancelled; + private boolean cacheEnabled = false; private SensorContextTester(Path moduleBaseDir) { this.settings = new MapSettings(); @@ -409,6 +414,35 @@ public class SensorContextTester implements SensorContext { } @Override + public WriteCache nextCache() { + return writeCache; + } + + public void setNextCache(WriteCache writeCache) { + this.writeCache = writeCache; + } + + @Override + public ReadCache previousAnalysisCache() { + return readCache; + } + + + public void setPreviousAnalysisCache(ReadCache cache) { + this.readCache = cache; + } + + @Override + public boolean isCacheEnabled() { + return cacheEnabled; + } + + + public void setCacheEnabled(boolean enabled) { + this.cacheEnabled = enabled; + } + + @Override public NewSignificantCode newSignificantCode() { return new DefaultSignificantCode(sensorStorage); } diff --git a/sonar-plugin-api-impl/src/test/java/org/sonar/api/batch/sensor/internal/SensorContextTesterTest.java b/sonar-plugin-api-impl/src/test/java/org/sonar/api/batch/sensor/internal/SensorContextTesterTest.java index eb61b3a55d4..12e42d9b3ff 100644 --- a/sonar-plugin-api-impl/src/test/java/org/sonar/api/batch/sensor/internal/SensorContextTesterTest.java +++ b/sonar-plugin-api-impl/src/test/java/org/sonar/api/batch/sensor/internal/SensorContextTesterTest.java @@ -36,6 +36,8 @@ import org.sonar.api.batch.rule.ActiveRules; import org.sonar.api.batch.rule.Severity; import org.sonar.api.batch.rule.internal.ActiveRulesBuilder; import org.sonar.api.batch.rule.internal.NewActiveRule; +import org.sonar.api.batch.sensor.cache.ReadCache; +import org.sonar.api.batch.sensor.cache.WriteCache; import org.sonar.api.batch.sensor.error.AnalysisError; import org.sonar.api.batch.sensor.error.NewAnalysisError; import org.sonar.api.batch.sensor.highlighting.TypeOfText; @@ -51,6 +53,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.tuple; import static org.assertj.core.data.MapEntry.entry; +import static org.mockito.Mockito.mock; public class SensorContextTesterTest { @@ -82,6 +85,24 @@ public class SensorContextTesterTest { } @Test + public void testPluginCache() { + assertThat(tester.nextCache()).isNull(); + assertThat(tester.previousAnalysisCache()).isNull(); + assertThat(tester.isCacheEnabled()).isFalse(); + + ReadCache readCache = mock(ReadCache.class); + WriteCache writeCache = mock(WriteCache.class); + + tester.setPreviousAnalysisCache(readCache); + tester.setNextCache(writeCache); + tester.setCacheEnabled(true); + + assertThat(tester.nextCache()).isEqualTo(writeCache); + assertThat(tester.previousAnalysisCache()).isEqualTo(readCache); + assertThat(tester.isCacheEnabled()).isTrue(); + } + + @Test public void testActiveRules() { NewActiveRule activeRule = new NewActiveRule.Builder() .setRuleKey(RuleKey.of("foo", "bar")) diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/Beta.java b/sonar-plugin-api/src/main/java/org/sonar/api/Beta.java new file mode 100644 index 00000000000..228515acfc5 --- /dev/null +++ b/sonar-plugin-api/src/main/java/org/sonar/api/Beta.java @@ -0,0 +1,40 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.api; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * This annotations marks API as experimental. + * API marked with it can be subject to breaking changes or removal in any future version without compliance with deprecation policies. + */ +@Retention(RetentionPolicy.CLASS) +@Target({ + ElementType.ANNOTATION_TYPE, + ElementType.CONSTRUCTOR, + ElementType.FIELD, + ElementType.METHOD, + ElementType.TYPE}) +@Documented +public @interface Beta {} diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/SensorContext.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/SensorContext.java index d1a43ae930d..da9578d0c98 100644 --- a/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/SensorContext.java +++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/SensorContext.java @@ -20,11 +20,14 @@ package org.sonar.api.batch.sensor; import java.io.Serializable; +import org.sonar.api.Beta; import org.sonar.api.SonarRuntime; import org.sonar.api.batch.fs.FileSystem; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.InputModule; import org.sonar.api.batch.rule.ActiveRules; +import org.sonar.api.batch.sensor.cache.ReadCache; +import org.sonar.api.batch.sensor.cache.WriteCache; import org.sonar.api.batch.sensor.code.NewSignificantCode; import org.sonar.api.batch.sensor.coverage.NewCoverage; import org.sonar.api.batch.sensor.cpd.NewCpdTokens; @@ -218,4 +221,33 @@ public interface SensorContext { */ void markForPublishing(InputFile inputFile); + /** + * Access object to write cache that will be stored and made available in a future analysis. + * If cache is disabled, the methods in the returned object will have no effect. + * This API is experimental and can be changed or dropped at any time. + * @see #isCacheEnabled() + * @since 9.4 + */ + @Beta + WriteCache nextCache(); + + /** + * Access object to read cached data. The origin of the cached data is not specified and could come from a different branch. + * If cache is disabled, the methods in the returned object will have no effect. + * This API is experimental and can be changed or dropped at any time. + * @see #isCacheEnabled() + * @since 9.4 + */ + @Beta + ReadCache previousAnalysisCache(); + + /** + * Returns true if caching is enabled. + * This API is experimental and can be changed or dropped at any time. + * @see #nextCache() + * @see #previousAnalysisCache() + * @since 9.4 + */ + @Beta + boolean isCacheEnabled(); } diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/cache/ReadCache.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/cache/ReadCache.java new file mode 100644 index 00000000000..e9b497393be --- /dev/null +++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/cache/ReadCache.java @@ -0,0 +1,38 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.api.batch.sensor.cache; + +import java.io.InputStream; +import org.sonar.api.Beta; + +@Beta +public interface ReadCache { + /** + * Returns an input stream for the data cached with the key. + * It's the responsibility of the caller to close the stream. + * @throws IllegalArgumentException if cache doesn't contain key + */ + InputStream read(String key); + + /** + * Checks whether the cache contains a key + */ + boolean contains(String key); +} diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/cache/WriteCache.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/cache/WriteCache.java new file mode 100644 index 00000000000..1a6bf3ce032 --- /dev/null +++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/cache/WriteCache.java @@ -0,0 +1,44 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.api.batch.sensor.cache; + +import java.io.InputStream; +import org.sonar.api.Beta; + +@Beta +public interface WriteCache { + /** + * Save a new entry in the cache. The stream will be consumed immediately. + * @throws IllegalArgumentException if the cache already contains the key + */ + void write(String key, InputStream data); + + /** + * Save a new entry in the cache. + * @throws IllegalArgumentException if the cache already contains the key + */ + void write(String key, byte[] data); + + /** + * Copy a cached entry from the previous cache to the new one. + * @throws IllegalArgumentException if the previous cache doesn't contain given key or if this cache already contains the key + */ + void copyFromPrevious(String key); +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheEnabled.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheEnabled.java new file mode 100644 index 00000000000..b93150ae944 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheEnabled.java @@ -0,0 +1,35 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.cache; + +import org.sonar.api.config.Configuration; + +public class AnalysisCacheEnabled { + static final String PROP_KEY = "sonar.analysisCache.enabled"; + private final Configuration configuration; + + public AnalysisCacheEnabled(Configuration configuration) { + this.configuration = configuration; + } + + public boolean isEnabled() { + return configuration.getBoolean(PROP_KEY).orElse(false); + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheLoader.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheLoader.java new file mode 100644 index 00000000000..420a5d5f852 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheLoader.java @@ -0,0 +1,87 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.cache; + +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.util.Optional; +import java.util.zip.InflaterInputStream; +import org.sonar.api.scanner.fs.InputProject; +import org.sonar.core.util.Protobuf; +import org.sonar.scanner.bootstrap.DefaultScannerWsClient; +import org.sonar.scanner.protocol.internal.ScannerInternal; +import org.sonar.scanner.protocol.internal.ScannerInternal.AnalysisCacheMsg; +import org.sonar.scanner.scan.branch.BranchConfiguration; +import org.sonar.scanner.scan.branch.BranchType; +import org.sonarqube.ws.client.GetRequest; +import org.sonarqube.ws.client.WsResponse; + +/** + * Loads plugin cache into the local storage + */ +public class AnalysisCacheLoader { + static final String CONTENT_ENCODING = "Content-Encoding"; + static final String ACCEPT_ENCODING = "Accept-Encoding"; + private static final String URL = "api/scanner_cache/get"; + + private final DefaultScannerWsClient wsClient; + private final InputProject project; + private final BranchConfiguration branchConfiguration; + + public AnalysisCacheLoader(DefaultScannerWsClient wsClient, InputProject project, BranchConfiguration branchConfiguration) { + this.project = project; + this.branchConfiguration = branchConfiguration; + this.wsClient = wsClient; + } + + public Optional<AnalysisCacheMsg> load() { + String url = URL + "?project=" + project.key(); + if (branchConfiguration.branchType() == BranchType.BRANCH && branchConfiguration.branchName() != null) { + url = url + "&branch=" + branchConfiguration.branchName(); + } + + GetRequest request = new GetRequest(url).setHeader(ACCEPT_ENCODING, "gzip"); + + try (WsResponse response = wsClient.call(request)) { + if (response.code() == HttpURLConnection.HTTP_NOT_FOUND) { + return Optional.empty(); + } + try (InputStream is = response.contentStream()) { + Optional<String> contentEncoding = response.header(CONTENT_ENCODING); + if (contentEncoding.isPresent() && contentEncoding.get().equals("gzip")) { + return Optional.of(decompress(is)); + } else { + return Optional.of(Protobuf.read(is, AnalysisCacheMsg.parser())); + } + } catch (IOException e) { + throw new IllegalStateException("Failed to download cache", e); + } + } + } + + private static AnalysisCacheMsg decompress(InputStream is) { + try (InflaterInputStream iis = new InflaterInputStream(is)) { + return Protobuf.read(iis, ScannerInternal.AnalysisCacheMsg.parser()); + } catch (IOException e) { + throw new IllegalStateException("Failed to decompress plugin cache", e); + } + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheMemoryStorage.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheMemoryStorage.java new file mode 100644 index 00000000000..420bd1711a8 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheMemoryStorage.java @@ -0,0 +1,59 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.cache; + +import java.io.InputStream; +import javax.annotation.CheckForNull; +import javax.annotation.Nullable; +import org.sonar.scanner.protocol.internal.ScannerInternal.AnalysisCacheMsg; + +public class AnalysisCacheMemoryStorage implements AnalysisCacheStorage { + private final AnalysisCacheLoader loader; + @Nullable + private AnalysisCacheMsg cache; + + public AnalysisCacheMemoryStorage(AnalysisCacheLoader loader) { + this.loader = loader; + } + + @Override + @CheckForNull + public InputStream get(String key) { + if (cache == null) { + return null; + } + if (cache.containsMap(key)) { + return cache.getMapOrThrow(key).newInput(); + } + return null; + } + + @Override + public boolean contains(String key) { + if (cache == null) { + return false; + } + return cache.containsMap(key); + } + + public void load() { + cache = loader.load().orElse(null); + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheProvider.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheProvider.java new file mode 100644 index 00000000000..b340261f569 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheProvider.java @@ -0,0 +1,83 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.cache; + +import java.io.InputStream; +import java.util.Map; +import org.jetbrains.annotations.Nullable; +import org.sonar.api.batch.sensor.cache.ReadCache; +import org.springframework.context.annotation.Bean; + +import static java.util.Collections.emptyMap; + +public class AnalysisCacheProvider { + @Bean("ReadCache") + public ReadCache provideReader(AnalysisCacheEnabled analysisCacheEnabled, AnalysisCacheMemoryStorage storage) { + if (analysisCacheEnabled.isEnabled()) { + storage.load(); + return new ReadCacheImpl(storage); + } + return new NoOpReadCache(); + } + + @Bean("WriteCache") + public ScannerWriteCache provideWriter(AnalysisCacheEnabled analysisCacheEnabled, ReadCache readCache) { + if (analysisCacheEnabled.isEnabled()) { + return new WriteCacheImpl(readCache); + } + return new NoOpWriteCache(); + } + + + static class NoOpWriteCache implements ScannerWriteCache { + @Override + public void write(String s, InputStream inputStream) { + // no op + } + + @Override + public void write(String s, byte[] bytes) { + // no op + } + + @Override + public void copyFromPrevious(String s) { + // no op + } + + @Override + public Map<String, byte[]> getCache() { + return emptyMap(); + } + } + + static class NoOpReadCache implements ReadCache { + @Nullable + @Override + public InputStream read(String s) { + return null; + } + + @Override + public boolean contains(String s) { + return false; + } + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheStorage.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheStorage.java new file mode 100644 index 00000000000..5fb7765da38 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/AnalysisCacheStorage.java @@ -0,0 +1,30 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.cache; + +import java.io.InputStream; +import javax.annotation.CheckForNull; + +public interface AnalysisCacheStorage { + @CheckForNull + InputStream get(String key); + + boolean contains(String key); +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/ReadCacheImpl.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/ReadCacheImpl.java new file mode 100644 index 00000000000..4a94e0ae7b0 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/ReadCacheImpl.java @@ -0,0 +1,47 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.cache; + +import java.io.InputStream; +import org.sonar.api.batch.sensor.cache.ReadCache; + +import static org.sonar.api.utils.Preconditions.checkArgument; +import static org.sonar.api.utils.Preconditions.checkNotNull; + +public class ReadCacheImpl implements ReadCache { + private final AnalysisCacheStorage cache; + + public ReadCacheImpl(AnalysisCacheStorage storage) { + this.cache = storage; + } + + @Override + public InputStream read(String key) { + checkNotNull(key); + checkArgument(contains(key)); + return cache.get(key); + } + + @Override + public boolean contains(String key) { + checkNotNull(key); + return cache.contains(key); + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/ScannerWriteCache.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/ScannerWriteCache.java new file mode 100644 index 00000000000..4087db134b5 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/ScannerWriteCache.java @@ -0,0 +1,27 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.cache; + +import java.util.Map; +import org.sonar.api.batch.sensor.cache.WriteCache; + +public interface ScannerWriteCache extends WriteCache { + Map<String, byte[]> getCache(); +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/WriteCacheImpl.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/WriteCacheImpl.java new file mode 100644 index 00000000000..adeb6216d23 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cache/WriteCacheImpl.java @@ -0,0 +1,81 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.cache; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import org.sonar.api.batch.sensor.cache.ReadCache; + +import static java.util.Collections.unmodifiableMap; +import static org.sonar.api.utils.Preconditions.checkArgument; +import static org.sonar.api.utils.Preconditions.checkNotNull; + +public class WriteCacheImpl implements ScannerWriteCache { + private final ReadCache readCache; + private final Map<String, byte[]> cache = new HashMap<>(); + + public WriteCacheImpl(ReadCache readCache) { + this.readCache = readCache; + } + + @Override + public void write(String key, InputStream data) { + checkNotNull(data); + checkKey(key); + try { + byte[] arr = data.readAllBytes(); + cache.put(key, arr); + } catch (IOException e) { + throw new IllegalStateException("Failed to read stream", e); + } + } + + @Override + public void write(String key, byte[] data) { + checkNotNull(data); + checkKey(key); + cache.put(key, Arrays.copyOf(data, data.length)); + } + + @Override + public void copyFromPrevious(String key) { + checkArgument(readCache.contains(key), "Previous cache doesn't contain key '%s'", key); + checkKey(key); + + try { + cache.put(key, readCache.read(key).readAllBytes()); + } catch (IOException e) { + throw new IllegalStateException("Failed to read plugin cache for key " + key, e); + } + } + + @Override + public Map<String, byte[]> getCache() { + return unmodifiableMap(cache); + } + + private void checkKey(String key) { + checkNotNull(key); + checkArgument(!cache.containsKey(key), "Cache already contains key '%s'", key); + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/PluginCachePublisher.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/AnalysisCachePublisher.java index 048d95d1432..6daae79356d 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/PluginCachePublisher.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/AnalysisCachePublisher.java @@ -21,31 +21,32 @@ package org.sonar.scanner.report; import com.google.protobuf.ByteString; import java.util.Map; -import org.sonar.scanner.cache.PluginCacheEnabled; +import org.sonar.scanner.cache.AnalysisCacheEnabled; import org.sonar.scanner.cache.ScannerWriteCache; -import org.sonar.scanner.protocol.internal.ScannerInternal.PluginCacheMsg; +import org.sonar.scanner.protocol.internal.ScannerInternal; +import org.sonar.scanner.protocol.internal.ScannerInternal.AnalysisCacheMsg; import org.sonar.scanner.protocol.output.ScannerReportWriter; -public class PluginCachePublisher implements ReportPublisherStep { - private final PluginCacheEnabled pluginCacheEnabled; +public class AnalysisCachePublisher implements ReportPublisherStep { + private final AnalysisCacheEnabled analysisCacheEnabled; private final ScannerWriteCache cache; - public PluginCachePublisher(PluginCacheEnabled pluginCacheEnabled, ScannerWriteCache cache) { - this.pluginCacheEnabled = pluginCacheEnabled; + public AnalysisCachePublisher(AnalysisCacheEnabled analysisCacheEnabled, ScannerWriteCache cache) { + this.analysisCacheEnabled = analysisCacheEnabled; this.cache = cache; } @Override public void publish(ScannerReportWriter writer) { - if (!pluginCacheEnabled.isEnabled() || cache.getCache().isEmpty()) { + if (!analysisCacheEnabled.isEnabled() || cache.getCache().isEmpty()) { return; } - PluginCacheMsg.Builder pluginCacheMsg = PluginCacheMsg.newBuilder(); + AnalysisCacheMsg.Builder analysisCacheMsg = ScannerInternal.AnalysisCacheMsg.newBuilder(); for (Map.Entry<String, byte[]> entry : cache.getCache().entrySet()) { - pluginCacheMsg.putMap(entry.getKey(), ByteString.copyFrom(entry.getValue())); + analysisCacheMsg.putMap(entry.getKey(), ByteString.copyFrom(entry.getValue())); } - writer.writePluginCache(pluginCacheMsg.build()); + writer.writeAnalysisCache(analysisCacheMsg.build()); } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java index 888f5661e8c..325032da593 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java @@ -35,6 +35,7 @@ import org.sonar.core.config.ScannerProperties; import org.sonar.core.extension.CoreExtensionsInstaller; import org.sonar.core.language.LanguagesProvider; import org.sonar.core.metric.ScannerMetrics; +import org.sonar.core.platform.SpringComponentContainer; import org.sonar.scanner.DefaultFileLinesContextFactory; import org.sonar.scanner.ProjectInfo; import org.sonar.scanner.analysis.AnalysisTempFolderProvider; @@ -42,7 +43,10 @@ import org.sonar.scanner.bootstrap.ExtensionInstaller; import org.sonar.scanner.bootstrap.ExtensionMatcher; import org.sonar.scanner.bootstrap.GlobalAnalysisMode; import org.sonar.scanner.bootstrap.PostJobExtensionDictionary; -import org.sonar.core.platform.SpringComponentContainer; +import org.sonar.scanner.cache.AnalysisCacheEnabled; +import org.sonar.scanner.cache.AnalysisCacheLoader; +import org.sonar.scanner.cache.AnalysisCacheMemoryStorage; +import org.sonar.scanner.cache.AnalysisCacheProvider; import org.sonar.scanner.ci.CiConfigurationProvider; import org.sonar.scanner.ci.vendors.AppVeyor; import org.sonar.scanner.ci.vendors.AwsCodeBuild; @@ -84,6 +88,7 @@ import org.sonar.scanner.report.ChangedLinesPublisher; import org.sonar.scanner.report.ComponentsPublisher; import org.sonar.scanner.report.ContextPropertiesPublisher; import org.sonar.scanner.report.MetadataPublisher; +import org.sonar.scanner.report.AnalysisCachePublisher; import org.sonar.scanner.report.ReportPublisher; import org.sonar.scanner.report.SourcePublisher; import org.sonar.scanner.report.TestExecutionPublisher; @@ -128,8 +133,8 @@ import org.sonar.scanner.sensor.ProjectSensorsExecutor; import org.sonar.scm.git.GitScmSupport; import org.sonar.scm.svn.SvnScmSupport; -import static org.sonar.api.utils.Preconditions.checkNotNull; import static org.sonar.api.batch.InstantiationStrategy.PER_BATCH; +import static org.sonar.api.utils.Preconditions.checkNotNull; import static org.sonar.core.extension.CoreExtensionsInstaller.noExtensionFilter; import static org.sonar.scanner.bootstrap.ExtensionUtils.isDeprecatedScannerSide; import static org.sonar.scanner.bootstrap.ExtensionUtils.isInstantiationStrategy; @@ -166,6 +171,7 @@ public class SpringProjectScanContainer extends SpringComponentContainer { new ProjectPullRequestsProvider(), ProjectRepositoriesSupplier.class, new ProjectServerSettingsProvider(), + AnalysisCacheEnabled.class, // temp new AnalysisTempFolderProvider(), @@ -223,6 +229,11 @@ public class SpringProjectScanContainer extends SpringComponentContainer { ProjectCoverageAndDuplicationExclusions.class, + // Plugin cache + AnalysisCacheProvider.class, + AnalysisCacheMemoryStorage.class, + AnalysisCacheLoader.class, + // Report ReferenceBranchSupplier.class, ScannerMetrics.class, @@ -232,6 +243,7 @@ public class SpringProjectScanContainer extends SpringComponentContainer { ActiveRulesPublisher.class, AnalysisWarningsPublisher.class, ComponentsPublisher.class, + AnalysisCachePublisher.class, TestExecutionPublisher.class, SourcePublisher.class, ChangedLinesPublisher.class, diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java index 461fe146a76..86c54c41d69 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java @@ -25,9 +25,12 @@ import org.sonar.api.batch.fs.FileSystem; import org.sonar.api.batch.fs.InputModule; import org.sonar.api.batch.fs.internal.DefaultInputProject; import org.sonar.api.batch.rule.ActiveRules; +import org.sonar.api.batch.sensor.cache.ReadCache; +import org.sonar.api.batch.sensor.cache.WriteCache; import org.sonar.api.batch.sensor.internal.SensorStorage; import org.sonar.api.config.Configuration; import org.sonar.api.config.Settings; +import org.sonar.scanner.cache.AnalysisCacheEnabled; import org.sonar.scanner.scan.branch.BranchConfiguration; @ThreadSafe @@ -36,8 +39,9 @@ public class ModuleSensorContext extends ProjectSensorContext { private final InputModule module; public ModuleSensorContext(DefaultInputProject project, InputModule module, Configuration config, Settings mutableModuleSettings, FileSystem fs, ActiveRules activeRules, - SensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration) { - super(project, config, mutableModuleSettings, fs, activeRules, sensorStorage, sonarRuntime, branchConfiguration); + SensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration, + WriteCache writeCache, ReadCache readCache, AnalysisCacheEnabled analysisCacheEnabled) { + super(project, config, mutableModuleSettings, fs, activeRules, sensorStorage, sonarRuntime, branchConfiguration, writeCache, readCache, analysisCacheEnabled); this.module = module; } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java index fc78598b80a..c3cd0ac0521 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java @@ -29,6 +29,8 @@ import org.sonar.api.batch.fs.internal.DefaultInputFile; import org.sonar.api.batch.fs.internal.DefaultInputProject; import org.sonar.api.batch.rule.ActiveRules; import org.sonar.api.batch.sensor.SensorContext; +import org.sonar.api.batch.sensor.cache.ReadCache; +import org.sonar.api.batch.sensor.cache.WriteCache; import org.sonar.api.batch.sensor.code.NewSignificantCode; import org.sonar.api.batch.sensor.code.internal.DefaultSignificantCode; import org.sonar.api.batch.sensor.coverage.NewCoverage; @@ -53,6 +55,7 @@ import org.sonar.api.config.Configuration; import org.sonar.api.config.Settings; import org.sonar.api.scanner.fs.InputProject; import org.sonar.api.utils.Version; +import org.sonar.scanner.cache.AnalysisCacheEnabled; import org.sonar.scanner.scan.branch.BranchConfiguration; import org.sonar.scanner.sensor.noop.NoOpNewAnalysisError; @@ -69,9 +72,13 @@ public class ProjectSensorContext implements SensorContext { private final SonarRuntime sonarRuntime; private final Configuration config; private final boolean skipUnchangedFiles; + private final WriteCache writeCache; + private final ReadCache readCache; + private final AnalysisCacheEnabled analysisCacheEnabled; public ProjectSensorContext(DefaultInputProject project, Configuration config, Settings mutableSettings, FileSystem fs, ActiveRules activeRules, - SensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration) { + SensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration, WriteCache writeCache, ReadCache readCache, + AnalysisCacheEnabled analysisCacheEnabled) { this.project = project; this.config = config; this.mutableSettings = mutableSettings; @@ -79,6 +86,9 @@ public class ProjectSensorContext implements SensorContext { this.activeRules = activeRules; this.sensorStorage = sensorStorage; this.sonarRuntime = sonarRuntime; + this.writeCache = writeCache; + this.readCache = readCache; + this.analysisCacheEnabled = analysisCacheEnabled; this.skipUnchangedFiles = branchConfiguration.isPullRequest(); } @@ -184,6 +194,21 @@ public class ProjectSensorContext implements SensorContext { } @Override + public WriteCache nextCache() { + return writeCache; + } + + @Override + public ReadCache previousAnalysisCache() { + return readCache; + } + + @Override + public boolean isCacheEnabled() { + return analysisCacheEnabled.isEnabled(); + } + + @Override public NewSignificantCode newSignificantCode() { return new DefaultSignificantCode(sensorStorage); } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheEnabledTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheEnabledTest.java new file mode 100644 index 00000000000..a4877298d3e --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheEnabledTest.java @@ -0,0 +1,45 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.cache; + +import java.util.Optional; +import org.junit.Test; +import org.sonar.api.config.Configuration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.sonar.scanner.cache.AnalysisCacheEnabled.PROP_KEY; + +public class AnalysisCacheEnabledTest { + private final Configuration configuration = mock(Configuration.class); + private final AnalysisCacheEnabled analysisCacheEnabled = new AnalysisCacheEnabled(configuration); + + @Test + public void disabled_unless_property_set() { + assertThat(analysisCacheEnabled.isEnabled()).isFalse(); + } + + @Test + public void enabled_if_property_set() { + when(configuration.getBoolean(PROP_KEY)).thenReturn(Optional.of(true)); + assertThat(analysisCacheEnabled.isEnabled()).isTrue(); + } +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheLoaderTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheLoaderTest.java new file mode 100644 index 00000000000..8ccdcafdb2d --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheLoaderTest.java @@ -0,0 +1,97 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.cache; + +import com.google.protobuf.ByteString; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import java.util.zip.DeflaterInputStream; +import org.junit.Before; +import org.junit.Test; +import org.sonar.api.scanner.fs.InputProject; +import org.sonar.scanner.bootstrap.DefaultScannerWsClient; +import org.sonar.scanner.protocol.internal.ScannerInternal; +import org.sonar.scanner.protocol.internal.ScannerInternal.AnalysisCacheMsg; +import org.sonar.scanner.scan.branch.BranchConfiguration; +import org.sonarqube.ws.client.WsResponse; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.sonar.scanner.cache.AnalysisCacheLoader.CONTENT_ENCODING; + +public class AnalysisCacheLoaderTest { + private final WsResponse response = mock(WsResponse.class); + private final DefaultScannerWsClient wsClient = mock(DefaultScannerWsClient.class); + private final InputProject project = mock(InputProject.class); + private final BranchConfiguration branchConfiguration = mock(BranchConfiguration.class); + private final AnalysisCacheLoader loader = new AnalysisCacheLoader(wsClient, project, branchConfiguration); + + @Before + public void before() { + when(wsClient.call(any())).thenReturn(response); + } + + @Test + public void loads_content() throws IOException { + ScannerInternal.AnalysisCacheMsg expected = ScannerInternal.AnalysisCacheMsg.newBuilder() + .putMap("key", ByteString.copyFrom("value", StandardCharsets.UTF_8)) + .build(); + setResponse(expected); + AnalysisCacheMsg msg = loader.load().get(); + assertThat(msg).isEqualTo(expected); + } + + @Test + public void loads_compressed_content() throws IOException { + AnalysisCacheMsg expected = AnalysisCacheMsg.newBuilder() + .putMap("key", ByteString.copyFrom("value", StandardCharsets.UTF_8)) + .build(); + setCompressedResponse(expected); + AnalysisCacheMsg msg = loader.load().get(); + assertThat(msg).isEqualTo(expected); + } + + @Test + public void returns_empty_if_404() { + when(response.code()).thenReturn(404); + assertThat(loader.load()).isEmpty(); + } + + private void setResponse(AnalysisCacheMsg msg) throws IOException { + when(response.contentStream()).thenReturn(createInputStream(msg)); + } + + private void setCompressedResponse(AnalysisCacheMsg msg) throws IOException { + when(response.contentStream()).thenReturn(new DeflaterInputStream(createInputStream(msg))); + when(response.header(CONTENT_ENCODING)).thenReturn(Optional.of("gzip")); + } + + private InputStream createInputStream(AnalysisCacheMsg analysisCacheMsg) throws IOException { + ByteArrayOutputStream serialized = new ByteArrayOutputStream(analysisCacheMsg.getSerializedSize()); + analysisCacheMsg.writeTo(serialized); + return new ByteArrayInputStream(serialized.toByteArray()); + } +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheMemoryStorageTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheMemoryStorageTest.java new file mode 100644 index 00000000000..fb3a994cf57 --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheMemoryStorageTest.java @@ -0,0 +1,66 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.cache; + +import com.google.protobuf.ByteString; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import org.apache.commons.io.IOUtils; +import org.junit.Test; +import org.sonar.scanner.protocol.internal.ScannerInternal.AnalysisCacheMsg; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class AnalysisCacheMemoryStorageTest { + private final AnalysisCacheLoader loader = mock(AnalysisCacheLoader.class); + private final AnalysisCacheMemoryStorage storage = new AnalysisCacheMemoryStorage(loader); + + @Test + public void storage_loads_with_loader() throws IOException { + when(loader.load()).thenReturn(Optional.of(AnalysisCacheMsg.newBuilder() + .putMap("key1", ByteString.copyFrom("value1", StandardCharsets.UTF_8)) + .build())); + + storage.load(); + verify(loader).load(); + assertThat(IOUtils.toString(storage.get("key1"), StandardCharsets.UTF_8)).isEqualTo("value1"); + assertThat(storage.contains("key1")).isTrue(); + } + + @Test + public void get_returns_null_if_doesnt_contain_key() { + when(loader.load()).thenReturn(Optional.of(AnalysisCacheMsg.newBuilder().build())); + storage.load(); + assertThat(storage.contains("key1")).isFalse(); + assertThat(storage.get("key1")).isNull(); + } + + @Test + public void get_returns_null_if_no_cache() { + when(loader.load()).thenReturn(Optional.empty()); + storage.load(); + assertThat(storage.contains("key1")).isFalse(); + assertThat(storage.get("key1")).isNull(); + } +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheProviderTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheProviderTest.java new file mode 100644 index 00000000000..5435e815c5f --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/AnalysisCacheProviderTest.java @@ -0,0 +1,65 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.cache; + +import org.junit.Test; +import org.sonar.api.batch.sensor.cache.ReadCache; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class AnalysisCacheProviderTest { + + AnalysisCacheEnabled analysisCacheEnabled = mock(AnalysisCacheEnabled.class); + AnalysisCacheMemoryStorage storage = mock(AnalysisCacheMemoryStorage.class); + ReadCache readCache = mock(ReadCache.class); + AnalysisCacheProvider cacheProvider = new AnalysisCacheProvider(); + + @Test + public void provide_noop_reader_cache_when_disable() { + when(analysisCacheEnabled.isEnabled()).thenReturn(false); + var cache = cacheProvider.provideReader(analysisCacheEnabled, storage); + assertThat(cache).isInstanceOf(AnalysisCacheProvider.NoOpReadCache.class); + } + + @Test + public void provide_noop_writer_cache_when_disable() { + when(analysisCacheEnabled.isEnabled()).thenReturn(false); + var cache = cacheProvider.provideWriter(analysisCacheEnabled, readCache); + assertThat(cache).isInstanceOf(AnalysisCacheProvider.NoOpWriteCache.class); + } + + @Test + public void provide_real_reader_cache_when_enable() { + when(analysisCacheEnabled.isEnabled()).thenReturn(true); + var cache = cacheProvider.provideReader(analysisCacheEnabled, storage); + verify(storage).load(); + assertThat(cache).isInstanceOf(ReadCacheImpl.class); + } + + @Test + public void provide_real_writer_cache_when_enable() { + when(analysisCacheEnabled.isEnabled()).thenReturn(true); + var cache = cacheProvider.provideWriter(analysisCacheEnabled, readCache); + assertThat(cache).isInstanceOf(WriteCacheImpl.class); + } +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/ReadCacheImplTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/ReadCacheImplTest.java new file mode 100644 index 00000000000..5ccf36e6cfc --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/ReadCacheImplTest.java @@ -0,0 +1,52 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.cache; + +import java.io.InputStream; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ReadCacheImplTest { + private final AnalysisCacheStorage storage = mock(AnalysisCacheStorage.class); + private final ReadCacheImpl readCache = new ReadCacheImpl(storage); + + @Test + public void read_delegates_to_storage() { + InputStream is = mock(InputStream.class); + when(storage.get("key")).thenReturn(is); + when(storage.contains("key")).thenReturn(true); + assertThat(readCache.read("key")).isEqualTo(is); + } + + @Test + public void read_fails_if_key_not_found() { + assertThatThrownBy(() -> readCache.read("unknown")).isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void contains_delegates_to_storage() { + when(storage.contains("key")).thenReturn(true); + assertThat(readCache.contains("key")).isTrue(); + } +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/WriteCacheImplTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/WriteCacheImplTest.java new file mode 100644 index 00000000000..653ded84d72 --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/cache/WriteCacheImplTest.java @@ -0,0 +1,85 @@ +/* + * SonarQube + * Copyright (C) 2009-2022 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.cache; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.data.MapEntry.entry; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class WriteCacheImplTest { + private final ReadCacheImpl readCache = mock(ReadCacheImpl.class); + private final WriteCacheImpl writeCache = new WriteCacheImpl(readCache); + + @Test + public void write_bytes_adds_entries() { + byte[] b1 = new byte[] {1, 2, 3}; + byte[] b2 = new byte[] {3, 4}; + writeCache.write("key", b1); + writeCache.write("key2", b2); + + assertThat(writeCache.getCache()).containsOnly(entry("key", b1), entry("key2", b2)); + } + + @Test + public void write_inputStream_adds_entries() { + byte[] b1 = new byte[] {1, 2, 3}; + byte[] b2 = new byte[] {3, 4}; + writeCache.write("key", new ByteArrayInputStream(b1)); + writeCache.write("key2", new ByteArrayInputStream(b2)); + + assertThat(writeCache.getCache()).containsOnly(entry("key", b1), entry("key2", b2)); + } + + @Test + public void write_throws_IAE_if_writing_same_key_twice() { + byte[] b1 = new byte[] {1}; + byte[] b2 = new byte[] {2}; + + + writeCache.write("key", b1); + assertThatThrownBy(() -> writeCache.write("key", b2)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cache already contains key 'key'"); + } + + @Test + public void copyFromPrevious_throws_IAE_if_read_cache_doesnt_contain_key() { + assertThatThrownBy(() -> writeCache.copyFromPrevious("key")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Previous cache doesn't contain key 'key'"); + } + + @Test + public void copyFromPrevious_reads_from_readCache() { + byte[] b = new byte[] {1}; + InputStream value = new ByteArrayInputStream(b); + when(readCache.contains("key")).thenReturn(true); + when(readCache.read("key")).thenReturn(value); + writeCache.copyFromPrevious("key"); + + assertThat(writeCache.getCache()).containsOnly(entry("key", b)); + } +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/PluginCachePublisherTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/AnalysisCachePublisherTest.java index 5e23f6853f9..c79939c16af 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/PluginCachePublisherTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/AnalysisCachePublisherTest.java @@ -26,7 +26,7 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.sonar.scanner.cache.PluginCacheEnabled; +import org.sonar.scanner.cache.AnalysisCacheEnabled; import org.sonar.scanner.cache.ScannerWriteCache; import org.sonar.scanner.protocol.output.ScannerReportWriter; @@ -38,13 +38,13 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; -public class PluginCachePublisherTest { +public class AnalysisCachePublisherTest { @Rule public TemporaryFolder temp = new TemporaryFolder(); private final ScannerWriteCache writeCache = mock(ScannerWriteCache.class); - private final PluginCacheEnabled pluginCacheEnabled = mock(PluginCacheEnabled.class); - private final PluginCachePublisher publisher = new PluginCachePublisher(pluginCacheEnabled, writeCache); + private final AnalysisCacheEnabled analysisCacheEnabled = mock(AnalysisCacheEnabled.class); + private final AnalysisCachePublisher publisher = new AnalysisCachePublisher(analysisCacheEnabled, writeCache); private ScannerReportWriter scannerReportWriter; @@ -55,7 +55,7 @@ public class PluginCachePublisherTest { @Test public void publish_does_nothing_if_cache_not_enabled() { - when(pluginCacheEnabled.isEnabled()).thenReturn(false); + when(analysisCacheEnabled.isEnabled()).thenReturn(false); publisher.publish(scannerReportWriter); verifyNoInteractions(writeCache); assertThat(scannerReportWriter.getFileStructure().root()).isEmptyDirectory(); @@ -64,18 +64,18 @@ public class PluginCachePublisherTest { @Test public void publish_cache() { when(writeCache.getCache()).thenReturn(Map.of("key1", "value1".getBytes(StandardCharsets.UTF_8))); - when(pluginCacheEnabled.isEnabled()).thenReturn(true); + when(analysisCacheEnabled.isEnabled()).thenReturn(true); publisher.publish(scannerReportWriter); verify(writeCache, times(2)).getCache(); - assertThat(scannerReportWriter.getFileStructure().pluginCache()).exists(); + assertThat(scannerReportWriter.getFileStructure().analysisCache()).exists(); } @Test public void publish_empty_cache() { when(writeCache.getCache()).thenReturn(emptyMap()); - when(pluginCacheEnabled.isEnabled()).thenReturn(true); + when(analysisCacheEnabled.isEnabled()).thenReturn(true); publisher.publish(scannerReportWriter); verify(writeCache).getCache(); - assertThat(scannerReportWriter.getFileStructure().pluginCache()).doesNotExist(); + assertThat(scannerReportWriter.getFileStructure().analysisCache()).doesNotExist(); } } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ModuleSensorContextTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ModuleSensorContextTest.java index 5b886662ecf..1063283716e 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ModuleSensorContextTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ModuleSensorContextTest.java @@ -37,6 +37,9 @@ import org.sonar.api.config.internal.MapSettings; import org.sonar.api.internal.SonarRuntimeImpl; import org.sonar.api.measures.CoreMetrics; import org.sonar.api.utils.Version; +import org.sonar.scanner.cache.AnalysisCacheEnabled; +import org.sonar.scanner.cache.ReadCacheImpl; +import org.sonar.scanner.cache.WriteCacheImpl; import org.sonar.scanner.scan.branch.BranchConfiguration; import static org.assertj.core.api.Assertions.assertThat; @@ -55,6 +58,9 @@ public class ModuleSensorContextTest { private SensorStorage sensorStorage; private SonarRuntime runtime; private BranchConfiguration branchConfiguration; + private WriteCacheImpl writeCache; + private ReadCacheImpl readCache; + private AnalysisCacheEnabled analysisCacheEnabled; @Before public void prepare() throws Exception { @@ -66,18 +72,25 @@ public class ModuleSensorContextTest { settings = new MapSettings(); sensorStorage = mock(SensorStorage.class); branchConfiguration = mock(BranchConfiguration.class); + writeCache = mock(WriteCacheImpl.class); + readCache = mock(ReadCacheImpl.class); + analysisCacheEnabled = mock(AnalysisCacheEnabled.class); runtime = SonarRuntimeImpl.forSonarQube(Version.parse("5.5"), SonarQubeSide.SCANNER, SonarEdition.COMMUNITY); } @Test public void shouldProvideComponents() { - adaptor = new ModuleSensorContext(mock(DefaultInputProject.class), mock(InputModule.class), settings.asConfig(), settings, fs, activeRules, sensorStorage, runtime, branchConfiguration); + adaptor = new ModuleSensorContext(mock(DefaultInputProject.class), mock(InputModule.class), settings.asConfig(), settings, fs, activeRules, sensorStorage, runtime, + branchConfiguration, writeCache, readCache, analysisCacheEnabled); assertThat(adaptor.activeRules()).isEqualTo(activeRules); assertThat(adaptor.fileSystem()).isEqualTo(fs); assertThat(adaptor.getSonarQubeVersion()).isEqualTo(Version.parse("5.5")); assertThat(adaptor.runtime()).isEqualTo(runtime); assertThat(adaptor.canSkipUnchangedFiles()).isFalse(); + assertThat(adaptor.nextCache()).isEqualTo(writeCache); + assertThat(adaptor.previousAnalysisCache()).isEqualTo(readCache); + assertThat(adaptor.newIssue()).isNotNull(); assertThat(adaptor.newExternalIssue()).isNotNull(); assertThat(adaptor.newAdHocRule()).isNotNull(); @@ -90,7 +103,8 @@ public class ModuleSensorContextTest { @Test public void pull_request_can_skip_unchanged_files() { when(branchConfiguration.isPullRequest()).thenReturn(true); - adaptor = new ModuleSensorContext(mock(DefaultInputProject.class), mock(InputModule.class), settings.asConfig(), settings, fs, activeRules, sensorStorage, runtime, branchConfiguration); + adaptor = new ModuleSensorContext(mock(DefaultInputProject.class), mock(InputModule.class), settings.asConfig(), settings, fs, activeRules, sensorStorage, runtime, + branchConfiguration, writeCache, readCache, analysisCacheEnabled); assertThat(adaptor.canSkipUnchangedFiles()).isTrue(); } diff --git a/sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/FileStructure.java b/sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/FileStructure.java index 041bf6cbeff..d5bcf2a3a4d 100644 --- a/sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/FileStructure.java +++ b/sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/FileStructure.java @@ -66,7 +66,7 @@ public class FileStructure { return new File(dir, "metadata.pb"); } - public File pluginCache() { + public File analysisCache() { return new File(dir, "plugin-cache.pb"); } diff --git a/sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportReader.java b/sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportReader.java index aaa0190a802..5e1376a3f1c 100644 --- a/sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportReader.java +++ b/sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportReader.java @@ -81,12 +81,12 @@ public class ScannerReportReader { @CheckForNull public InputStream getPluginCache() { - File file = fileStructure.pluginCache(); + File file = fileStructure.analysisCache(); if (fileExists(file)) { try { - return new BufferedInputStream(new FileInputStream(fileStructure.pluginCache())); + return new BufferedInputStream(new FileInputStream(fileStructure.analysisCache())); } catch (FileNotFoundException e) { - throw new IllegalStateException("Unable to open file " + fileStructure.pluginCache(), e); + throw new IllegalStateException("Unable to open file " + fileStructure.analysisCache(), e); } } return null; diff --git a/sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportWriter.java b/sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportWriter.java index 88f16d14d19..1b67cd41ea6 100644 --- a/sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportWriter.java +++ b/sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/output/ScannerReportWriter.java @@ -95,8 +95,8 @@ public class ScannerReportWriter { return file; } - public File writePluginCache(ScannerInternal.PluginCacheMsg cacheMsg) { - File file = fileStructure.pluginCache(); + public File writeAnalysisCache(ScannerInternal.AnalysisCacheMsg cacheMsg) { + File file = fileStructure.analysisCache(); Protobuf.writeGzip(cacheMsg, file); return file; } diff --git a/sonar-scanner-protocol/src/main/protobuf/scanner_internal.proto b/sonar-scanner-protocol/src/main/protobuf/scanner_internal.proto index b0d38058c4c..4a9f0c53a58 100644 --- a/sonar-scanner-protocol/src/main/protobuf/scanner_internal.proto +++ b/sonar-scanner-protocol/src/main/protobuf/scanner_internal.proto @@ -3,6 +3,6 @@ syntax = "proto3"; option java_package = "org.sonar.scanner.protocol.internal"; option optimize_for = SPEED; -message PluginCacheMsg { +message AnalysisCacheMsg { map<string, bytes> map = 1; } diff --git a/sonar-scanner-protocol/src/test/java/org/sonar/scanner/protocol/output/ScannerReportReaderTest.java b/sonar-scanner-protocol/src/test/java/org/sonar/scanner/protocol/output/ScannerReportReaderTest.java index 58333d108cf..1269c63c540 100644 --- a/sonar-scanner-protocol/src/test/java/org/sonar/scanner/protocol/output/ScannerReportReaderTest.java +++ b/sonar-scanner-protocol/src/test/java/org/sonar/scanner/protocol/output/ScannerReportReaderTest.java @@ -35,7 +35,8 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.sonar.core.util.CloseableIterator; import org.sonar.core.util.Protobuf; -import org.sonar.scanner.protocol.internal.ScannerInternal.PluginCacheMsg; +import org.sonar.scanner.protocol.internal.ScannerInternal; +import org.sonar.scanner.protocol.internal.ScannerInternal.AnalysisCacheMsg; import org.sonar.scanner.protocol.output.ScannerReport.Measure.StringValue; import org.sonar.scanner.protocol.output.ScannerReport.SyntaxHighlightingRule.HighlightingType; @@ -214,13 +215,13 @@ public class ScannerReportReaderTest { @Test public void read_plugin_cache() throws IOException { ScannerReportWriter writer = new ScannerReportWriter(dir); - writer.writePluginCache(PluginCacheMsg.newBuilder() + writer.writeAnalysisCache(ScannerInternal.AnalysisCacheMsg.newBuilder() .putMap("key", ByteString.copyFrom("data", UTF_8)) .build()); ScannerReportReader reader = new ScannerReportReader(dir); - PluginCacheMsg cache = Protobuf.read(new GZIPInputStream(reader.getPluginCache()), PluginCacheMsg.parser()); + AnalysisCacheMsg cache = Protobuf.read(new GZIPInputStream(reader.getPluginCache()), ScannerInternal.AnalysisCacheMsg.parser()); assertThat(cache.getMapMap()).containsOnly(new AbstractMap.SimpleEntry<>("key", ByteString.copyFrom("data", UTF_8))); } |