Browse Source

Revert "SONAR-21195 Enhance scanner engine to download only required plugins"

This reverts commit 981e6b8595.
tags/10.4.0.87286
Matteo Mara 4 months ago
parent
commit
6c6de5ad4f
30 changed files with 415 additions and 1093 deletions
  1. 5
    10
      plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/Xoo2.java
  2. 0
    7
      plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java
  3. 1
    1
      settings.gradle
  4. 10
    14
      sonar-core/src/main/java/org/sonar/core/metric/ScannerMetrics.java
  5. 0
    5
      sonar-core/src/main/java/org/sonar/core/platform/ExtensionContainer.java
  6. 0
    10
      sonar-core/src/main/java/org/sonar/core/platform/ListContainer.java
  7. 0
    31
      sonar-core/src/main/java/org/sonar/core/platform/SpringComponentContainer.java
  8. 1
    24
      sonar-core/src/test/java/org/sonar/core/metric/ScannerMetricsTest.java
  9. 1
    11
      sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/ScannerMediumTester.java
  10. 0
    3
      sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java
  11. 3
    2
      sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/NoLanguagesPluginsMediumIT.java
  12. 3
    8
      sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ExtensionInstaller.java
  13. 2
    15
      sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java
  14. 10
    73
      sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java
  15. 11
    47
      sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java
  16. 5
    4
      sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringGlobalContainer.java
  17. 0
    346
      sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java
  18. 0
    83
      sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakeLanguagesRepository.java
  19. 1
    12
      sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java
  20. 13
    57
      sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/DefaultLanguagesRepository.java
  21. 0
    63
      sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/SupportedLanguageDto.java
  22. 1
    3
      sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/settings/AbstractSettingsLoader.java
  23. 252
    29
      sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java
  24. 5
    32
      sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/ScannerPluginInstallerTest.java
  25. 13
    50
      sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/ScannerPluginRepositoryTest.java
  26. 68
    96
      sonar-scanner-engine/src/test/java/org/sonar/scanner/repository/language/DefaultLanguagesRepositoryTest.java
  27. 0
    27
      sonar-scanner-engine/src/test/java/org/sonar/scanner/repository/settings/AbstractSettingsLoaderTest.java
  28. 8
    8
      sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java
  29. 2
    6
      sonar-scanner-engine/src/test/resources/org/sonar/scanner/bootstrap/ScannerPluginInstallerTest/installed-plugins-ws.json
  30. 0
    16
      sonar-scanner-engine/src/test/resources/org/sonar/scanner/repository/language/DefaultLanguageRepositoryTest/languages-ws.json

+ 5
- 10
plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/Xoo2.java View File

@@ -19,22 +19,17 @@
*/
package org.sonar.xoo;

import java.util.Arrays;
import org.sonar.api.config.Configuration;
import org.sonar.api.resources.Language;

public class Xoo2 implements Language {

public static final String KEY = "xoo2";
public static final String NAME = "Xoo2";
public static final String FILE_SUFFIXES_KEY = "sonar.xoo2.file.suffixes";
public static final String DEFAULT_FILE_SUFFIXES = ".xoo2";
public static final String FILE_SUFFIX = ".xoo2";

private final Configuration configuration;

public Xoo2(Configuration configuration) {
this.configuration = configuration;
}
private static final String[] XOO_SUFFIXES = {
FILE_SUFFIX
};

@Override
public String getKey() {
@@ -48,7 +43,7 @@ public class Xoo2 implements Language {

@Override
public String[] getFileSuffixes() {
return Arrays.stream(configuration.getStringArray(FILE_SUFFIXES_KEY)).filter(s -> s != null && !s.trim().isEmpty()).toArray(String[]::new);
return XOO_SUFFIXES;
}

@Override

+ 0
- 7
plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java View File

@@ -105,13 +105,6 @@ public class XooPlugin implements Plugin {
.onQualifiers(Qualifiers.PROJECT)
.multiValues(true)
.build(),
PropertyDefinition.builder(Xoo2.FILE_SUFFIXES_KEY)
.defaultValue(Xoo2.DEFAULT_FILE_SUFFIXES)
.name("File suffixes")
.description("Comma-separated list of suffixes for files to analyze. To not filter, leave the list empty.")
.subCategory("General")
.onQualifiers(Qualifiers.PROJECT)
.build(),
// Used by DuplicationsTest and IssueFilterOnCommonRulesTest. If not declared it is not returned by api/settings
PropertyDefinition.builder("sonar.cpd.xoo.minimumTokens")
.onQualifiers(Qualifiers.PROJECT)

+ 1
- 1
settings.gradle View File

@@ -74,4 +74,4 @@ buildCache {
local {
enabled = !isCiServer
}
}
}

+ 10
- 14
sonar-core/src/main/java/org/sonar/core/metric/ScannerMetrics.java View File

@@ -19,11 +19,14 @@
*/
package org.sonar.core.metric;

import com.google.common.collect.ImmutableSet;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.annotation.concurrent.Immutable;
import org.sonar.api.ce.ComputeEngineSide;
import org.sonar.api.measures.Metric;
import org.sonar.api.measures.Metrics;
@@ -58,11 +61,12 @@ import static org.sonar.api.measures.CoreMetrics.TEST_FAILURES;
* <p/>
* Scanners should not send other metrics, and the Compute Engine should not allow other metrics.
*/
@Immutable
@ComputeEngineSide
@ScannerSide
public class ScannerMetrics {

private static final Set<Metric> ALLOWED_CORE_METRICS = Set.of(
private static final Set<Metric> ALLOWED_CORE_METRICS = ImmutableSet.of(
GENERATED_LINES,
NCLOC,
NCLOC_DATA,
@@ -91,7 +95,7 @@ public class ScannerMetrics {

EXECUTABLE_LINES_DATA);

private Set<Metric> metrics;
private final Set<Metric> metrics;

@Autowired(required = false)
public ScannerMetrics() {
@@ -99,9 +103,8 @@ public class ScannerMetrics {
}

@Autowired(required = false)
public ScannerMetrics(List<Metrics> metricsRepositories) {
this.metrics = ALLOWED_CORE_METRICS;
addPluginMetrics(metricsRepositories);
public ScannerMetrics(Metrics[] metricsRepositories) {
this.metrics = Stream.concat(getPluginMetrics(metricsRepositories), ALLOWED_CORE_METRICS.stream()).collect(Collectors.toSet());
}

/**
@@ -112,15 +115,8 @@ public class ScannerMetrics {
return metrics;
}

/**
* Adds the given metrics to the set of allowed metrics
*/
public void addPluginMetrics(List<Metrics> metricsRepositories) {
this.metrics = Stream.concat(getPluginMetrics(metricsRepositories.stream()), this.metrics.stream()).collect(Collectors.toSet());
}

private static Stream<Metric> getPluginMetrics(Stream<Metrics> metricsStream) {
return metricsStream
private static Stream<Metric> getPluginMetrics(Metrics[] metricsRepositories) {
return Arrays.stream(metricsRepositories)
.map(Metrics::getMetrics)
.filter(Objects::nonNull)
.flatMap(List::stream);

+ 0
- 5
sonar-core/src/main/java/org/sonar/core/platform/ExtensionContainer.java View File

@@ -19,7 +19,6 @@
*/
package org.sonar.core.platform;

import java.util.List;
import java.util.Set;
import javax.annotation.CheckForNull;
import javax.annotation.Nullable;
@@ -37,10 +36,6 @@ public interface ExtensionContainer extends Container {

Set<Class<?>> getWebApiV2ConfigurationClasses();

<T> T getParentComponentByType(Class<T> type);

<T> List<T> getParentComponentsByType(Class<T> type);

@CheckForNull
ExtensionContainer getParent();
}

+ 0
- 10
sonar-core/src/main/java/org/sonar/core/platform/ListContainer.java View File

@@ -101,16 +101,6 @@ public class ListContainer implements ExtensionContainer {
return webConfigurationClasses;
}

@Override
public <T> T getParentComponentByType(Class<T> type) {
throw new UnsupportedOperationException();
}

@Override
public <T> List<T> getParentComponentsByType(Class<T> type) {
throw new UnsupportedOperationException();
}

@Override
public ExtensionContainer getParent() {
throw new UnsupportedOperationException();

+ 0
- 31
sonar-core/src/main/java/org/sonar/core/platform/SpringComponentContainer.java View File

@@ -60,10 +60,6 @@ public class SpringComponentContainer implements StartableContainer {
this(parent, parent.propertyDefinitions, emptyList(), new LazyUnlessStartableStrategy());
}

protected SpringComponentContainer(SpringComponentContainer parent, List<?> externalExtensions) {
this(parent, parent.propertyDefinitions, externalExtensions, new LazyUnlessStartableStrategy());
}

protected SpringComponentContainer(SpringComponentContainer parent, SpringInitStrategy initStrategy) {
this(parent, parent.propertyDefinitions, emptyList(), initStrategy);
}
@@ -85,15 +81,6 @@ public class SpringComponentContainer implements StartableContainer {
add(propertyDefs);
}

//TODO: To be removed, added for moving on with the non matching LanguagesRepository beans
public void addIfMissing(Object object, Class<?> objectType) {
try {
getParentComponentByType(objectType);
} catch (IllegalStateException e) {
add(object);
}
}

/**
* Beans need to have a unique name, otherwise they'll override each other.
* The strategy is:
@@ -133,24 +120,6 @@ public class SpringComponentContainer implements StartableContainer {
return Set.copyOf(webConfigurationClasses);
}

@Override
public <T> T getParentComponentByType(Class<T> type) {
if (parent == null) {
throw new IllegalStateException("No parent container");
} else {
return parent.getComponentByType(type);
}
}

@Override
public <T> List<T> getParentComponentsByType(Class<T> type) {
if (parent == null) {
throw new IllegalStateException("No parent container");
} else {
return parent.getComponentsByType(type);
}
}

private <T> void registerInstance(T instance) {
Supplier<T> supplier = () -> instance;
Class<T> clazz = (Class<T>) instance.getClass();

+ 1
- 24
sonar-core/src/test/java/org/sonar/core/metric/ScannerMetricsTest.java View File

@@ -56,23 +56,8 @@ public class ScannerMetricsTest {
assertThat(metrics).isEqualTo(okMetrics.getMetrics());
}

@Test
public void should_add_new_plugin_metrics() {
Metrics fakeMetrics = new FakeMetrics();
Metrics fakeMetrics2 = new FakeMetrics2();

ScannerMetrics underTest = new ScannerMetrics(List.of(fakeMetrics));
assertThat(underTest.getMetrics()).hasSize(24);
assertThat(underTest.getMetrics()).containsAll(fakeMetrics.getMetrics());

underTest.addPluginMetrics(List.of( fakeMetrics, fakeMetrics2 ));
assertThat(underTest.getMetrics()).hasSize(25);
assertThat(underTest.getMetrics()).containsAll(fakeMetrics.getMetrics());
assertThat(underTest.getMetrics()).containsAll(fakeMetrics2.getMetrics());
}

private static List<Metric> metrics(Metrics... metrics) {
return new ArrayList<>(new ScannerMetrics(Arrays.asList(metrics)).getMetrics());
return new ArrayList<>(new ScannerMetrics(metrics).getMetrics());
}

private static class FakeMetrics implements Metrics {
@@ -83,12 +68,4 @@ public class ScannerMetricsTest {
new Metric.Builder("key2", "name2", Metric.ValueType.FLOAT).create());
}
}

private static class FakeMetrics2 implements Metrics {
@Override
public List<Metric> getMetrics() {
return Arrays.asList(
new Metric.Builder("key3", "name1", Metric.ValueType.INT).create());
}
}
}

+ 1
- 11
sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/ScannerMediumTester.java View File

@@ -105,7 +105,6 @@ public class ScannerMediumTester extends ExternalResource {
private final FakeActiveRulesLoader activeRules = new FakeActiveRulesLoader();
private final FakeSonarRuntime sonarRuntime = new FakeSonarRuntime();
private final CeTaskReportDataHolder reportMetadataHolder = new CeTaskReportDataHolderExt();
private final FakeLanguagesRepository languagesRepository = new FakeLanguagesRepository();
private LogOutput logOutput = null;

private static void createWorkingDirs() throws IOException {
@@ -283,14 +282,6 @@ public class ScannerMediumTester extends ExternalResource {
return builder;
}

public void addLanguage(String key, String name, String... suffixes) {
languagesRepository.addLanguage(key, name, suffixes, new String[0]);
}

public void addLanguage(String key, String name, boolean publishAllFiles, String... suffixes) {
languagesRepository.addLanguage(key, name, suffixes, new String[0], publishAllFiles);
}

public static class AnalysisBuilder {
private final Map<String, String> taskProperties = new HashMap<>();
private final ScannerMediumTester tester;
@@ -322,8 +313,7 @@ public class ScannerMediumTester extends ExternalResource {
tester.analysisCacheLoader,
tester.sonarRuntime,
tester.reportMetadataHolder,
result,
tester.languagesRepository);
result);
if (tester.logOutput != null) {
builder.setLogOutput(tester.logOutput);
} else {

+ 0
- 3
sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java View File

@@ -866,8 +866,6 @@ public class FileSystemMediumIT {
File srcDir = new File(baseDir, "src");
srcDir.mkdir();

tester.addLanguage("xoo3", "xoo3",false, ".xoo3");

writeFile(srcDir, "sample.xoo3", "Sample xoo\ncontent");
writeFile(srcDir, "sample2.xoo3", "Sample xoo 2\ncontent");

@@ -899,7 +897,6 @@ public class FileSystemMediumIT {

assertThat(result.inputFiles()).hasSize(2);

tester.addLanguage("xoo2", "xoo2", ".xoo");
AnalysisBuilder analysisBuilder = tester.newAnalysis()
.properties(builder
.put("sonar.lang.patterns.xoo2", "**/*.xoo")

+ 3
- 2
sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/NoLanguagesPluginsMediumIT.java View File

@@ -26,6 +26,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.sonar.scanner.mediumtest.ScannerMediumTester;
import org.springframework.beans.factory.BeanCreationException;

import static org.assertj.core.api.Assertions.assertThatThrownBy;

@@ -43,8 +44,8 @@ public class NoLanguagesPluginsMediumIT {
assertThatThrownBy(() -> tester
.newAnalysis(new File(projectDir, "sonar-project.properties"))
.execute())
.isInstanceOf(IllegalStateException.class)
.hasMessage("No language plugins are installed.");
.isInstanceOf(BeanCreationException.class)
.hasRootCauseMessage("No language plugins are installed.");
}

private File copyProject(String path) throws Exception {

+ 3
- 8
sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ExtensionInstaller.java View File

@@ -19,7 +19,6 @@
*/
package org.sonar.scanner.bootstrap;

import java.util.Collection;
import javax.annotation.Nullable;
import org.sonar.api.Plugin;
import org.sonar.api.SonarRuntime;
@@ -48,13 +47,7 @@ public class ExtensionInstaller {
}

// plugin extensions
installExtensionsForPlugins(container, matcher, pluginRepository.getPluginInfos());

return this;
}

public void installExtensionsForPlugins(ExtensionContainer container, ExtensionMatcher matcher, Collection<PluginInfo> pluginInfos) {
for (PluginInfo pluginInfo : pluginInfos) {
for (PluginInfo pluginInfo : pluginRepository.getPluginInfos()) {
Plugin plugin = pluginRepository.getPluginInstance(pluginInfo.getKey());
Plugin.Context context = new PluginContextImpl.Builder()
.setSonarRuntime(sonarRuntime)
@@ -66,6 +59,8 @@ public class ExtensionInstaller {
doInstall(container, matcher, pluginInfo, extension);
}
}

return this;
}

private static void doInstall(ExtensionContainer container, ExtensionMatcher matcher, @Nullable PluginInfo pluginInfo, Object extension) {

+ 2
- 15
sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java View File

@@ -21,28 +21,15 @@ package org.sonar.scanner.bootstrap;

import java.util.List;
import java.util.Map;
import java.util.Set;

public interface PluginInstaller {

/**
* Loads/downloads all plugins that are installed on the server.
* @return information about all installed plugins, grouped by key
*/
Map<String, ScannerPlugin> installAllPlugins();

/**
* Gets the list of plugins that are not required for any specific languages and downloads them if not
* Gets the list of plugins installed on server and downloads them if not
* already in local cache.
* @return information about all installed plugins, grouped by key
*/
Map<String, ScannerPlugin> installRequiredPlugins();

/**
* Loads/downloads plugins that are required for the given languageKeys.
* @return information about any plugins installed by this call, grouped by key
*/
Map<String, ScannerPlugin> installPluginsForLanguages(Set<String> languageKeys);
Map<String, ScannerPlugin> installRemotes();

/**
* Used only by medium tests.

+ 10
- 73
sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java View File

@@ -22,14 +22,11 @@ package org.sonar.scanner.bootstrap;
import com.google.gson.Gson;
import java.io.File;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Predicate;
import javax.annotation.Nullable;
import org.sonar.api.utils.log.Logger;
import org.sonar.api.utils.log.Loggers;
@@ -51,58 +48,21 @@ public class ScannerPluginInstaller implements PluginInstaller {
private final PluginFiles pluginFiles;
private final DefaultScannerWsClient wsClient;

private List<InstalledPlugin> availablePlugins;

public ScannerPluginInstaller(PluginFiles pluginFiles, DefaultScannerWsClient wsClient) {
this.pluginFiles = pluginFiles;
this.wsClient = wsClient;
}

@Override
public Map<String, ScannerPlugin> installAllPlugins() {
LOG.info("Loading all plugins");
return installPlugins(p -> true).installedPluginsByKey;
}

@Override
public Map<String, ScannerPlugin> installRequiredPlugins() {
LOG.info("Loading required plugins");
InstallResult result = installPlugins(p -> p.getRequiredForLanguages() == null || p.getRequiredForLanguages().isEmpty());

LOG.debug("Plugins not loaded because they are optional: {}", result.skippedPlugins);

return result.installedPluginsByKey;
}

@Override
public Map<String, ScannerPlugin> installPluginsForLanguages(Set<String> languageKeys) {
LOG.info("Loading plugins for detected languages");
LOG.debug("Detected languages: {}", languageKeys);
InstallResult result = installPlugins(
p -> p.getRequiredForLanguages() != null && !Collections.disjoint(p.getRequiredForLanguages(), languageKeys)
);

List<InstalledPlugin> skippedLanguagePlugins = result.skippedPlugins.stream()
.filter(p -> p.getRequiredForLanguages() != null && !p.getRequiredForLanguages().isEmpty()).toList();
LOG.debug("Optional language-specific plugins not loaded: {}", skippedLanguagePlugins);

return result.installedPluginsByKey;
}

private InstallResult installPlugins(Predicate<InstalledPlugin> pluginFilter) {
if (this.availablePlugins == null) {
this.availablePlugins = listInstalledPlugins();
}

public Map<String, ScannerPlugin> installRemotes() {
Profiler profiler = Profiler.create(LOG).startInfo("Load/download plugins");
try {
InstallResult result = new InstallResult();
Loaded loaded = loadPlugins(result, pluginFilter);
Map<String, ScannerPlugin> result = new HashMap<>();
Loaded loaded = loadPlugins(result);
if (!loaded.ok) {
// retry once, a plugin may have been uninstalled during downloads
this.availablePlugins = listInstalledPlugins();
result.installedPluginsByKey.clear();
loaded = loadPlugins(result, pluginFilter);
result.clear();
loaded = loadPlugins(result);
if (!loaded.ok) {
throw new IllegalStateException(format("Fail to download plugin [%s]. Not found.", loaded.notFoundPlugin));
}
@@ -113,23 +73,16 @@ public class ScannerPluginInstaller implements PluginInstaller {
}
}

private Loaded loadPlugins(InstallResult result, Predicate<InstalledPlugin> pluginFilter) {
List<InstalledPlugin> pluginsToInstall = availablePlugins.stream()
.filter(pluginFilter).toList();

for (InstalledPlugin plugin : pluginsToInstall) {
private Loaded loadPlugins(Map<String, ScannerPlugin> result) {
for (InstalledPlugin plugin : listInstalledPlugins()) {
Optional<File> jarFile = pluginFiles.get(plugin);
if (jarFile.isEmpty()) {
return new Loaded(false, plugin.key);
}

PluginInfo info = PluginInfo.create(jarFile.get());
result.installedPluginsByKey.put(info.getKey(), new ScannerPlugin(plugin.key, plugin.updatedAt, PluginType.valueOf(plugin.type), info));
result.put(info.getKey(), new ScannerPlugin(plugin.key, plugin.updatedAt, PluginType.valueOf(plugin.type), info));
}

result.skippedPlugins = availablePlugins.stream()
.filter(Predicate.not(pluginFilter)).toList();

return new Loaded(true, null);
}

@@ -144,7 +97,7 @@ public class ScannerPluginInstaller implements PluginInstaller {
/**
* Gets information about the plugins installed on server (filename, checksum)
*/
private List<InstalledPlugin> listInstalledPlugins() {
private InstalledPlugin[] listInstalledPlugins() {
Profiler profiler = Profiler.create(LOG).startInfo("Load plugins index");
GetRequest getRequest = new GetRequest(PLUGINS_WS_URL);
InstalledPlugins installedPlugins;
@@ -158,13 +111,8 @@ public class ScannerPluginInstaller implements PluginInstaller {
return installedPlugins.plugins;
}

private static class InstallResult {
Map<String, ScannerPlugin> installedPluginsByKey = new HashMap<>();
List<InstalledPlugin> skippedPlugins = new ArrayList<>();
}

private static class InstalledPlugins {
List<InstalledPlugin> plugins;
InstalledPlugin[] plugins;

public InstalledPlugins() {
// http://stackoverflow.com/a/18645370/229031
@@ -176,21 +124,10 @@ public class ScannerPluginInstaller implements PluginInstaller {
String hash;
long updatedAt;
String type;
private Set<String> requiredForLanguages;

public InstalledPlugin() {
// http://stackoverflow.com/a/18645370/229031
}

public Set<String> getRequiredForLanguages() {
return requiredForLanguages;
}

@Override
public String toString() {
return key;
}

}

private static class Loaded {

+ 11
- 47
sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java View File

@@ -20,16 +20,14 @@
package org.sonar.scanner.bootstrap;

import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import javax.annotation.CheckForNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonar.api.Plugin;
import org.sonar.api.Startable;
import org.sonar.api.config.Configuration;
import org.sonar.core.platform.ExplodedPlugin;
import org.sonar.core.platform.PluginClassLoader;
import org.sonar.core.platform.PluginInfo;
@@ -37,7 +35,6 @@ import org.sonar.core.platform.PluginJarExploder;
import org.sonar.core.platform.PluginRepository;
import org.sonar.core.plugin.PluginType;

import static java.util.stream.Collectors.toMap;
import static org.sonar.api.utils.Preconditions.checkState;

/**
@@ -50,33 +47,22 @@ public class ScannerPluginRepository implements PluginRepository, Startable {
private final PluginJarExploder pluginJarExploder;
private final PluginClassLoader loader;

private final Configuration properties;

private Map<String, Plugin> pluginInstancesByKeys;
private Map<String, ScannerPlugin> pluginsByKeys;
private Map<ClassLoader, String> keysByClassLoader;
private boolean shouldLoadAllPluginsOnStart;

public ScannerPluginRepository(PluginInstaller installer, PluginJarExploder pluginJarExploder, PluginClassLoader loader, Configuration properties) {
public ScannerPluginRepository(PluginInstaller installer, PluginJarExploder pluginJarExploder, PluginClassLoader loader) {
this.installer = installer;
this.pluginJarExploder = pluginJarExploder;
this.loader = loader;
this.properties = properties;
}

@Override
public void start() {
shouldLoadAllPluginsOnStart = properties.getBoolean("sonar.plugins.loadAll").orElse(false);
if (shouldLoadAllPluginsOnStart) {
LOG.warn("sonar.plugins.loadAll is true, so ALL available plugins will be downloaded");
pluginsByKeys = new HashMap<>(installer.installAllPlugins());
} else {
pluginsByKeys = new HashMap<>(installer.installRequiredPlugins());
}

Map<String, ExplodedPlugin> explodedPluginsByKey = pluginsByKeys.entrySet().stream()
.collect(toMap(Map.Entry::getKey, e -> pluginJarExploder.explode(e.getValue().getInfo())));
pluginInstancesByKeys = new HashMap<>(loader.load(explodedPluginsByKey));
pluginsByKeys = new HashMap<>(installer.installRemotes());
Map<String, ExplodedPlugin> explodedPLuginsByKey = pluginsByKeys.entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> pluginJarExploder.explode(e.getValue().getInfo())));
pluginInstancesByKeys = new HashMap<>(loader.load(explodedPLuginsByKey));

// this part is only used by medium tests
for (Object[] localPlugin : installer.installLocals()) {
@@ -91,29 +77,7 @@ public class ScannerPluginRepository implements PluginRepository, Startable {
keysByClassLoader.put(e.getValue().getClass().getClassLoader(), e.getKey());
}

logPlugins(pluginsByKeys.values());
}

public Collection<PluginInfo> installPluginsForLanguages(Set<String> languageKeys) {
if (shouldLoadAllPluginsOnStart) {
return Collections.emptySet();
}

var languagePluginsByKeys = new HashMap<>(installer.installPluginsForLanguages(languageKeys));

pluginsByKeys.putAll(languagePluginsByKeys);

Map<String, ExplodedPlugin> explodedPluginsByKey = languagePluginsByKeys.entrySet().stream()
.collect(toMap(Map.Entry::getKey, e -> pluginJarExploder.explode(e.getValue().getInfo())));
pluginInstancesByKeys.putAll(new HashMap<>(loader.load(explodedPluginsByKey)));

keysByClassLoader = new HashMap<>();
for (Map.Entry<String, Plugin> e : pluginInstancesByKeys.entrySet()) {
keysByClassLoader.put(e.getValue().getClass().getClassLoader(), e.getKey());
}

logPlugins(languagePluginsByKeys.values());
return languagePluginsByKeys.values().stream().map(ScannerPlugin::getInfo).toList();
logPlugins();
}

@CheckForNull
@@ -121,12 +85,12 @@ public class ScannerPluginRepository implements PluginRepository, Startable {
return keysByClassLoader.get(cl);
}

private static void logPlugins(Collection<ScannerPlugin> plugins) {
if (plugins.isEmpty()) {
private void logPlugins() {
if (pluginsByKeys.isEmpty()) {
LOG.debug("No plugins loaded");
} else {
LOG.debug("Plugins loaded:");
for (ScannerPlugin p : plugins) {
LOG.debug("Plugins:");
for (ScannerPlugin p : pluginsByKeys.values()) {
LOG.debug(" * {} {} ({})", p.getName(), p.getVersion(), p.getKey());
}
}

+ 5
- 4
sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringGlobalContainer.java View File

@@ -54,8 +54,9 @@ import org.sonar.scanner.repository.DefaultMetricsRepositoryLoader;
import org.sonar.scanner.repository.DefaultNewCodePeriodLoader;
import org.sonar.scanner.repository.MetricsRepositoryProvider;
import org.sonar.scanner.repository.settings.DefaultGlobalSettingsLoader;
import org.sonar.scanner.scan.SpringProjectScanContainer;

@Priority(4)
@Priority(3)
public class SpringGlobalContainer extends SpringComponentContainer {
private static final Logger LOG = LoggerFactory.getLogger(SpringGlobalContainer.class);
private final Map<String, String> scannerProperties;
@@ -119,7 +120,7 @@ public class SpringGlobalContainer extends SpringComponentContainer {

@Override
protected void doAfterStart() {
installRequiredPlugins();
installPlugins();
loadCoreExtensions();

long startTime = System.currentTimeMillis();
@@ -135,12 +136,12 @@ public class SpringGlobalContainer extends SpringComponentContainer {
throw MessageException.of("The preview mode, along with the 'sonar.analysis.mode' parameter, is no more supported. You should stop using this parameter.");
}
getComponentByType(RuntimeJavaVersion.class).checkJavaVersion();
new SpringScannerContainer(this).execute();
new SpringProjectScanContainer(this).execute();

LOG.info("Analysis total time: {}", formatTime(System.currentTimeMillis() - startTime));
}

private void installRequiredPlugins() {
private void installPlugins() {
PluginRepository pluginRepository = getComponentByType(PluginRepository.class);
for (PluginInfo pluginInfo : pluginRepository.getPluginInfos()) {
Plugin instance = pluginRepository.getPluginInstance(pluginInfo.getKey());

+ 0
- 346
sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java View File

@@ -1,346 +0,0 @@
/*
* SonarQube
* Copyright (C) 2009-2023 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.scanner.bootstrap;

import javax.annotation.Nullable;
import javax.annotation.Priority;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonar.api.batch.fs.internal.FileMetadata;
import org.sonar.api.batch.fs.internal.SensorStrategy;
import org.sonar.api.batch.rule.CheckFactory;
import org.sonar.api.batch.sensor.issue.internal.DefaultNoSonarFilter;
import org.sonar.api.scan.filesystem.PathResolver;
import org.sonar.api.utils.MessageException;
import org.sonar.core.extension.CoreExtensionsInstaller;
import org.sonar.core.metric.ScannerMetrics;
import org.sonar.core.platform.SpringComponentContainer;
import org.sonar.scanner.DefaultFileLinesContextFactory;
import org.sonar.scanner.ProjectInfo;
import org.sonar.scanner.analysis.AnalysisTempFolderProvider;
import org.sonar.scanner.cache.AnalysisCacheEnabled;
import org.sonar.scanner.cache.AnalysisCacheMemoryStorage;
import org.sonar.scanner.cache.AnalysisCacheProvider;
import org.sonar.scanner.cache.DefaultAnalysisCacheLoader;
import org.sonar.scanner.ci.CiConfigurationProvider;
import org.sonar.scanner.ci.vendors.AppVeyor;
import org.sonar.scanner.ci.vendors.AwsCodeBuild;
import org.sonar.scanner.ci.vendors.AzureDevops;
import org.sonar.scanner.ci.vendors.Bamboo;
import org.sonar.scanner.ci.vendors.BitbucketPipelines;
import org.sonar.scanner.ci.vendors.Bitrise;
import org.sonar.scanner.ci.vendors.Buildkite;
import org.sonar.scanner.ci.vendors.CircleCi;
import org.sonar.scanner.ci.vendors.CirrusCi;
import org.sonar.scanner.ci.vendors.CodeMagic;
import org.sonar.scanner.ci.vendors.DroneCi;
import org.sonar.scanner.ci.vendors.GithubActions;
import org.sonar.scanner.ci.vendors.GitlabCi;
import org.sonar.scanner.ci.vendors.Jenkins;
import org.sonar.scanner.ci.vendors.SemaphoreCi;
import org.sonar.scanner.ci.vendors.TravisCi;
import org.sonar.scanner.cpd.CpdExecutor;
import org.sonar.scanner.cpd.CpdSettings;
import org.sonar.scanner.cpd.index.SonarCpdBlockIndex;
import org.sonar.scanner.issue.IssueFilters;
import org.sonar.scanner.issue.IssuePublisher;
import org.sonar.scanner.issue.ignore.EnforceIssuesFilter;
import org.sonar.scanner.issue.ignore.IgnoreIssuesFilter;
import org.sonar.scanner.issue.ignore.pattern.IssueExclusionPatternInitializer;
import org.sonar.scanner.issue.ignore.pattern.IssueInclusionPatternInitializer;
import org.sonar.scanner.issue.ignore.scanner.IssueExclusionsLoader;
import org.sonar.scanner.report.ActiveRulesPublisher;
import org.sonar.scanner.report.AnalysisCachePublisher;
import org.sonar.scanner.report.AnalysisContextReportPublisher;
import org.sonar.scanner.report.AnalysisWarningsPublisher;
import org.sonar.scanner.report.CeTaskReportDataHolder;
import org.sonar.scanner.report.ChangedLinesPublisher;
import org.sonar.scanner.report.ComponentsPublisher;
import org.sonar.scanner.report.JavaArchitectureInformationProvider;
import org.sonar.scanner.report.MetadataPublisher;
import org.sonar.scanner.report.ReportPublisher;
import org.sonar.scanner.report.ScannerFileStructureProvider;
import org.sonar.scanner.report.SourcePublisher;
import org.sonar.scanner.report.TestExecutionPublisher;
import org.sonar.scanner.repository.ContextPropertiesCache;
import org.sonar.scanner.repository.DefaultProjectRepositoriesLoader;
import org.sonar.scanner.repository.DefaultQualityProfileLoader;
import org.sonar.scanner.repository.ProjectRepositoriesProvider;
import org.sonar.scanner.repository.QualityProfilesProvider;
import org.sonar.scanner.repository.ReferenceBranchSupplier;
import org.sonar.scanner.repository.language.DefaultLanguagesRepository;
import org.sonar.scanner.repository.language.LanguagesRepository;
import org.sonar.scanner.repository.settings.DefaultProjectSettingsLoader;
import org.sonar.scanner.rule.ActiveRulesProvider;
import org.sonar.scanner.rule.DefaultActiveRulesLoader;
import org.sonar.scanner.rule.QProfileVerifier;
import org.sonar.scanner.scan.DeprecatedPropertiesWarningGenerator;
import org.sonar.scanner.scan.InputModuleHierarchyProvider;
import org.sonar.scanner.scan.InputProjectProvider;
import org.sonar.scanner.scan.ModuleIndexer;
import org.sonar.scanner.scan.MutableProjectReactorProvider;
import org.sonar.scanner.scan.MutableProjectSettings;
import org.sonar.scanner.scan.ProjectBuildersExecutor;
import org.sonar.scanner.scan.ProjectConfigurationProvider;
import org.sonar.scanner.scan.ProjectLock;
import org.sonar.scanner.scan.ProjectReactorBuilder;
import org.sonar.scanner.scan.ProjectReactorValidator;
import org.sonar.scanner.scan.ProjectServerSettingsProvider;
import org.sonar.scanner.scan.ScanProperties;
import org.sonar.scanner.scan.SonarGlobalPropertiesFilter;
import org.sonar.scanner.scan.SpringProjectScanContainer;
import org.sonar.scanner.scan.WorkDirectoriesInitializer;
import org.sonar.scanner.scan.branch.BranchConfiguration;
import org.sonar.scanner.scan.branch.BranchConfigurationProvider;
import org.sonar.scanner.scan.branch.BranchType;
import org.sonar.scanner.scan.branch.ProjectBranchesProvider;
import org.sonar.scanner.scan.filesystem.DefaultProjectFileSystem;
import org.sonar.scanner.scan.filesystem.FileIndexer;
import org.sonar.scanner.scan.filesystem.InputComponentStore;
import org.sonar.scanner.scan.filesystem.LanguageDetection;
import org.sonar.scanner.scan.filesystem.MetadataGenerator;
import org.sonar.scanner.scan.filesystem.ProjectCoverageAndDuplicationExclusions;
import org.sonar.scanner.scan.filesystem.ProjectExclusionFilters;
import org.sonar.scanner.scan.filesystem.ProjectFileIndexer;
import org.sonar.scanner.scan.filesystem.ScannerComponentIdGenerator;
import org.sonar.scanner.scan.filesystem.StatusDetection;
import org.sonar.scanner.scan.measure.DefaultMetricFinder;
import org.sonar.scanner.scm.ScmChangedFilesProvider;
import org.sonar.scanner.scm.ScmConfiguration;
import org.sonar.scanner.scm.ScmPublisher;
import org.sonar.scanner.scm.ScmRevisionImpl;
import org.sonar.scanner.sensor.DefaultSensorStorage;
import org.sonar.scanner.sensor.ExecutingSensorContext;
import org.sonar.scanner.sensor.ProjectSensorContext;
import org.sonar.scanner.sensor.ProjectSensorOptimizer;
import org.sonar.scanner.sensor.UnchangedFilesHandler;
import org.sonar.scm.git.GitScmSupport;
import org.sonar.scm.svn.SvnScmSupport;

import static org.sonar.api.batch.InstantiationStrategy.PER_BATCH;
import static org.sonar.core.extension.CoreExtensionsInstaller.noExtensionFilter;
import static org.sonar.scanner.bootstrap.ExtensionUtils.isDeprecatedScannerSide;
import static org.sonar.scanner.bootstrap.ExtensionUtils.isInstantiationStrategy;
import static org.sonar.scanner.bootstrap.ExtensionUtils.isScannerSide;

@Priority(3)
public class SpringScannerContainer extends SpringComponentContainer {
private static final Logger LOG = LoggerFactory.getLogger(SpringScannerContainer.class);

public SpringScannerContainer(SpringComponentContainer globalContainer) {
super(globalContainer);
}

@Override
protected void doBeforeStart() {
addScannerExtensions();
addComponents();
}

private void addScannerExtensions() {
getParentComponentByType(CoreExtensionsInstaller.class)
.install(this, noExtensionFilter(), extension -> getScannerProjectExtensionsFilter().accept(extension));
getParentComponentByType(ExtensionInstaller.class)
.install(this, getScannerProjectExtensionsFilter());
}

private void addComponents() {
add(
ScanProperties.class,
ProjectReactorBuilder.class,
WorkDirectoriesInitializer.class,
new MutableProjectReactorProvider(),
ProjectBuildersExecutor.class,
ProjectLock.class,
ProjectReactorValidator.class,
ProjectInfo.class,
new BranchConfigurationProvider(),
new ProjectBranchesProvider(),
ProjectRepositoriesProvider.class,
new ProjectServerSettingsProvider(),
AnalysisCacheEnabled.class,
DeprecatedPropertiesWarningGenerator.class,

// temp
new AnalysisTempFolderProvider(),

// file system
ModuleIndexer.class,
InputComponentStore.class,
PathResolver.class,
new InputProjectProvider(),
new InputModuleHierarchyProvider(),
ScannerComponentIdGenerator.class,
new ScmChangedFilesProvider(),
StatusDetection.class,
LanguageDetection.class,
MetadataGenerator.class,
FileMetadata.class,
FileIndexer.class,
ProjectFileIndexer.class,
ProjectExclusionFilters.class,

// rules
new ActiveRulesProvider(),
new QualityProfilesProvider(),
CheckFactory.class,
QProfileVerifier.class,

// issues
DefaultNoSonarFilter.class,
IssueFilters.class,
IssuePublisher.class,

// metrics
DefaultMetricFinder.class,

// issue exclusions
IssueInclusionPatternInitializer.class,
IssueExclusionPatternInitializer.class,
IssueExclusionsLoader.class,
EnforceIssuesFilter.class,
IgnoreIssuesFilter.class,

// context
ContextPropertiesCache.class,

SensorStrategy.class,

MutableProjectSettings.class,
SonarGlobalPropertiesFilter.class,
ProjectConfigurationProvider.class,

ProjectCoverageAndDuplicationExclusions.class,

// Plugin cache
AnalysisCacheProvider.class,
AnalysisCacheMemoryStorage.class,
DefaultAnalysisCacheLoader.class,

// Report
ReferenceBranchSupplier.class,
ScannerMetrics.class,
JavaArchitectureInformationProvider.class,
ReportPublisher.class,
ScannerFileStructureProvider.class,
AnalysisContextReportPublisher.class,
MetadataPublisher.class,
ActiveRulesPublisher.class,
ComponentsPublisher.class,
AnalysisCachePublisher.class,
TestExecutionPublisher.class,
SourcePublisher.class,
ChangedLinesPublisher.class,
AnalysisWarningsPublisher.class,

CeTaskReportDataHolder.class,

// Cpd
CpdExecutor.class,
CpdSettings.class,
SonarCpdBlockIndex.class,

// SCM
ScmConfiguration.class,
ScmPublisher.class,
ScmRevisionImpl.class,

// Sensors
DefaultSensorStorage.class,
DefaultFileLinesContextFactory.class,
ProjectSensorContext.class,
ProjectSensorOptimizer.class,
ExecutingSensorContext.class,

UnchangedFilesHandler.class,

// Filesystem
DefaultProjectFileSystem.class,

// CI
new CiConfigurationProvider(),
AppVeyor.class,
AwsCodeBuild.class,
AzureDevops.class,
Bamboo.class,
BitbucketPipelines.class,
Bitrise.class,
Buildkite.class,
CircleCi.class,
CirrusCi.class,
DroneCi.class,
GithubActions.class,
CodeMagic.class,
GitlabCi.class,
Jenkins.class,
SemaphoreCi.class,
TravisCi.class
);

add(GitScmSupport.getObjects());
add(SvnScmSupport.getObjects());

add(DefaultProjectSettingsLoader.class,
DefaultActiveRulesLoader.class,
DefaultQualityProfileLoader.class,
DefaultProjectRepositoriesLoader.class);

addIfMissing(DefaultLanguagesRepository.class, LanguagesRepository.class);

}

static ExtensionMatcher getScannerProjectExtensionsFilter() {
return extension -> {
if (isDeprecatedScannerSide(extension)) {
return isInstantiationStrategy(extension, PER_BATCH);
}
return isScannerSide(extension);
};
}

@Override
protected void doAfterStart() {
ScanProperties properties = getComponentByType(ScanProperties.class);
properties.validate();

properties.get("sonar.branch").ifPresent(deprecatedBranch -> {
throw MessageException.of("The 'sonar.branch' parameter is no longer supported. You should stop using it. " +
"Branch analysis is available in Developer Edition and above. See https://www.sonarsource.com/plans-and-pricing/developer/ for more information.");
});

BranchConfiguration branchConfig = getComponentByType(BranchConfiguration.class);
if (branchConfig.branchType() == BranchType.PULL_REQUEST) {
LOG.info("Pull request {} for merge into {} from {}", branchConfig.pullRequestKey(), pullRequestBaseToDisplayName(branchConfig.targetBranchName()),
branchConfig.branchName());
} else if (branchConfig.branchName() != null) {
LOG.info("Branch name: {}", branchConfig.branchName());
}

getComponentByType(DeprecatedPropertiesWarningGenerator.class).execute();

getComponentByType(ProjectFileIndexer.class).index();
new SpringProjectScanContainer(this).execute();
}

private static String pullRequestBaseToDisplayName(@Nullable String pullRequestBase) {
return pullRequestBase != null ? pullRequestBase : "default branch";
}

}

+ 0
- 83
sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakeLanguagesRepository.java View File

@@ -1,83 +0,0 @@
/*
* SonarQube
* Copyright (C) 2009-2023 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.scanner.mediumtest;

import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
import javax.annotation.Priority;
import org.jetbrains.annotations.Nullable;
import org.sonar.api.resources.Languages;
import org.sonar.scanner.repository.language.Language;
import org.sonar.scanner.repository.language.LanguagesRepository;
import org.sonar.scanner.repository.language.SupportedLanguageDto;

@Priority(1)
public class FakeLanguagesRepository implements LanguagesRepository {

private final Map<String, Language> languageMap = new HashMap<>();

public FakeLanguagesRepository() {
languageMap.put("xoo", new Language(new FakeLanguage("xoo", "xoo", new String[] { ".xoo" }, new String[0], true)));
}

public FakeLanguagesRepository(Languages languages) {
for (org.sonar.api.resources.Language language : languages.all()) {
languageMap.put(language.getKey(), new Language(new FakeLanguage(language.getKey(), language.getName(), language.getFileSuffixes(), language.filenamePatterns(), true)));
}
}

@Nullable
@Override
public Language get(String languageKey) {
return languageMap.get(languageKey);
}

@Override
public Collection<Language> all() {
return languageMap.values().stream()
// sorted for test consistency
.sorted(Comparator.comparing(Language::key)).toList();
}

public void addLanguage(String key, String name, String[] suffixes, String[] filenamePatterns) {
languageMap.put(key, new Language(new FakeLanguage(key, name, suffixes, filenamePatterns, true)));
}

public void addLanguage(String key, String name, String[] suffixes, String[] filenamePatterns, boolean publishAllFiles) {
languageMap.put(key, new Language(new FakeLanguage(key, name, suffixes, filenamePatterns, publishAllFiles)));
}

private static class FakeLanguage extends SupportedLanguageDto {

private final boolean publishAllFiles;

public FakeLanguage(String key, String name, String[] fileSuffixes, String[] filenamePatterns, boolean publishAllFiles) {
super(key, name, fileSuffixes, filenamePatterns);
this.publishAllFiles = publishAllFiles;
}

@Override
public boolean publishAllFiles() {
return publishAllFiles;
}
}
}

+ 1
- 12
sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java View File

@@ -24,7 +24,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.annotation.Priority;
import org.sonar.api.Plugin;
import org.sonar.core.platform.PluginInfo;
@@ -49,17 +48,7 @@ public class FakePluginInstaller implements PluginInstaller {
}

@Override
public Map<String, ScannerPlugin> installAllPlugins() {
return pluginsByKeys;
}

@Override
public Map<String, ScannerPlugin> installRequiredPlugins() {
return pluginsByKeys;
}

@Override
public Map<String, ScannerPlugin> installPluginsForLanguages(Set<String> languageKeys) {
public Map<String, ScannerPlugin> installRemotes() {
return pluginsByKeys;
}


+ 13
- 57
sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/DefaultLanguagesRepository.java View File

@@ -19,74 +19,31 @@
*/
package org.sonar.scanner.repository.language;

import com.google.gson.Gson;
import java.io.Reader;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import javax.annotation.CheckForNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.concurrent.Immutable;
import org.sonar.api.Startable;
import org.sonar.api.config.Configuration;
import org.sonar.api.resources.Languages;
import org.sonar.scanner.bootstrap.DefaultScannerWsClient;
import org.sonarqube.ws.client.GetRequest;

/**
* Languages repository using {@link Languages}
* @since 4.4
*/
@Immutable
public class DefaultLanguagesRepository implements LanguagesRepository, Startable {
private static final Logger LOG = LoggerFactory.getLogger(DefaultLanguagesRepository.class);
private static final String LANGUAGES_WS_URL = "/api/languages/list";
private static final Map<String, String> PROPERTY_FRAGMENT_MAP = Map.of(
"js", "javascript",
"ts", "typescript",
"py", "python",
"web", "html"
);

private final Map<String, Language> languages = new HashMap<>();
private final DefaultScannerWsClient wsClient;
private final Configuration properties;
private final Languages languages;

public DefaultLanguagesRepository(DefaultScannerWsClient wsClient, Configuration properties) {
this.wsClient = wsClient;
this.properties = properties;
public DefaultLanguagesRepository(Languages languages) {
this.languages = languages;
}

@Override
public void start() {
GetRequest getRequest = new GetRequest(LANGUAGES_WS_URL);
LanguagesWSResponse response;
try (Reader reader = wsClient.call(getRequest).contentReader()) {
response = new Gson().fromJson(reader, LanguagesWSResponse.class);
} catch (Exception e) {
throw new IllegalStateException("Fail to parse response of " + LANGUAGES_WS_URL, e);
if (languages.all().length == 0) {
throw new IllegalStateException("No language plugins are installed.");
}

languages.putAll(response.languages.stream()
.map(this::populateFileSuffixesAndPatterns)
.collect(Collectors.toMap(Language::key, Function.identity())));
}

private Language populateFileSuffixesAndPatterns(SupportedLanguageDto lang) {
String propertyFragment = PROPERTY_FRAGMENT_MAP.getOrDefault(lang.getKey(), lang.getKey());
lang.setFileSuffixes(properties.getStringArray(String.format("sonar.%s.file.suffixes", propertyFragment)));
lang.setFilenamePatterns(properties.getStringArray(String.format("sonar.%s.file.patterns", propertyFragment)));
if (lang.filenamePatterns() == null && lang.getFileSuffixes() == null) {
LOG.debug("Language '{}' cannot be detected as it has neither suffixes nor patterns.", lang.getName());
}
return new Language(lang);
}

private String[] getFileSuffixes(String languageKey) {
String propName = String.format("sonar.%s.file.suffixes", PROPERTY_FRAGMENT_MAP.getOrDefault(languageKey, languageKey));
return properties.getStringArray(propName);
}

/**
@@ -95,7 +52,8 @@ public class DefaultLanguagesRepository implements LanguagesRepository, Startabl
@Override
@CheckForNull
public Language get(String languageKey) {
return languages.get(languageKey);
org.sonar.api.resources.Language language = languages.get(languageKey);
return language != null ? new Language(language) : null;
}

/**
@@ -103,7 +61,9 @@ public class DefaultLanguagesRepository implements LanguagesRepository, Startabl
*/
@Override
public Collection<Language> all() {
return languages.values();
return Arrays.stream(languages.all())
.map(Language::new)
.toList();
}

@Override
@@ -111,8 +71,4 @@ public class DefaultLanguagesRepository implements LanguagesRepository, Startabl
// nothing to do
}

private static class LanguagesWSResponse {
List<SupportedLanguageDto> languages;
}

}

+ 0
- 63
sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/SupportedLanguageDto.java View File

@@ -1,63 +0,0 @@
/*
* SonarQube
* Copyright (C) 2009-2023 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.scanner.repository.language;

public class SupportedLanguageDto implements org.sonar.api.resources.Language {
private String key;
private String name;
private String[] fileSuffixes;
private String[] filenamePatterns;

public SupportedLanguageDto(String key, String name, String[] fileSuffixes, String[] filenamePatterns) {
this.key = key;
this.name = name;
this.fileSuffixes = fileSuffixes;
this.filenamePatterns = filenamePatterns;
}

@Override
public String getKey() {
return key;
}

@Override
public String getName() {
return name;
}

@Override
public String[] getFileSuffixes() {
return fileSuffixes;
}

public void setFileSuffixes(String[] fileSuffixes) {
this.fileSuffixes = fileSuffixes;
}

@Override
public String[] filenamePatterns() {
return filenamePatterns;
}

public void setFilenamePatterns(String[] filenamePatterns) {
this.filenamePatterns = filenamePatterns;
}

}

+ 1
- 3
sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/settings/AbstractSettingsLoader.java View File

@@ -74,9 +74,7 @@ public abstract class AbstractSettingsLoader {
static Map<String, String> toMap(List<Settings.Setting> settingsList) {
Map<String, String> result = new LinkedHashMap<>();
for (Settings.Setting s : settingsList) {
// we need the "*.file.suffixes" and "*.file.patterns" properties for language detection
// see DefaultLanguagesRepository.populateFileSuffixesAndPatterns()
if (!s.getInherited() || s.getKey().endsWith(".file.suffixes") || s.getKey().endsWith(".file.patterns")) {
if (!s.getInherited()) {
switch (s.getValueOneOfCase()) {
case VALUE:
result.put(s.getKey(), s.getValue());

+ 252
- 29
sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java View File

@@ -19,42 +19,124 @@
*/
package org.sonar.scanner.scan;

import java.util.Collection;
import java.util.Set;
import javax.annotation.Nullable;
import javax.annotation.Priority;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonar.api.Plugin;
import org.sonar.api.batch.fs.internal.DefaultInputModule;
import org.sonar.api.measures.Metrics;
import org.sonar.api.resources.Languages;
import org.sonar.api.batch.fs.internal.FileMetadata;
import org.sonar.api.batch.fs.internal.SensorStrategy;
import org.sonar.api.batch.rule.CheckFactory;
import org.sonar.api.batch.sensor.issue.internal.DefaultNoSonarFilter;
import org.sonar.api.resources.ResourceTypes;
import org.sonar.api.scan.filesystem.PathResolver;
import org.sonar.api.utils.MessageException;
import org.sonar.core.config.ScannerProperties;
import org.sonar.core.extension.CoreExtensionsInstaller;
import org.sonar.core.language.LanguagesProvider;
import org.sonar.core.metric.ScannerMetrics;
import org.sonar.core.platform.PluginInfo;
import org.sonar.core.platform.SpringComponentContainer;
import org.sonar.scanner.DefaultFileLinesContextFactory;
import org.sonar.scanner.ProjectInfo;
import org.sonar.scanner.analysis.AnalysisTempFolderProvider;
import org.sonar.scanner.bootstrap.ExtensionInstaller;
import org.sonar.scanner.bootstrap.ExtensionMatcher;
import org.sonar.scanner.bootstrap.GlobalAnalysisMode;
import org.sonar.scanner.bootstrap.PostJobExtensionDictionary;
import org.sonar.scanner.bootstrap.ScannerPluginRepository;
import org.sonar.scanner.cache.AnalysisCacheEnabled;
import org.sonar.scanner.cache.AnalysisCacheMemoryStorage;
import org.sonar.scanner.cache.AnalysisCacheProvider;
import org.sonar.scanner.cache.DefaultAnalysisCacheLoader;
import org.sonar.scanner.ci.CiConfigurationProvider;
import org.sonar.scanner.ci.vendors.AppVeyor;
import org.sonar.scanner.ci.vendors.AwsCodeBuild;
import org.sonar.scanner.ci.vendors.AzureDevops;
import org.sonar.scanner.ci.vendors.Bamboo;
import org.sonar.scanner.ci.vendors.BitbucketPipelines;
import org.sonar.scanner.ci.vendors.Bitrise;
import org.sonar.scanner.ci.vendors.Buildkite;
import org.sonar.scanner.ci.vendors.CircleCi;
import org.sonar.scanner.ci.vendors.CirrusCi;
import org.sonar.scanner.ci.vendors.CodeMagic;
import org.sonar.scanner.ci.vendors.DroneCi;
import org.sonar.scanner.ci.vendors.GithubActions;
import org.sonar.scanner.ci.vendors.GitlabCi;
import org.sonar.scanner.ci.vendors.Jenkins;
import org.sonar.scanner.ci.vendors.SemaphoreCi;
import org.sonar.scanner.ci.vendors.TravisCi;
import org.sonar.scanner.cpd.CpdExecutor;
import org.sonar.scanner.cpd.CpdSettings;
import org.sonar.scanner.cpd.index.SonarCpdBlockIndex;
import org.sonar.scanner.fs.InputModuleHierarchy;
import org.sonar.scanner.issue.IssueFilters;
import org.sonar.scanner.issue.IssuePublisher;
import org.sonar.scanner.issue.ignore.EnforceIssuesFilter;
import org.sonar.scanner.issue.ignore.IgnoreIssuesFilter;
import org.sonar.scanner.issue.ignore.pattern.IssueExclusionPatternInitializer;
import org.sonar.scanner.issue.ignore.pattern.IssueInclusionPatternInitializer;
import org.sonar.scanner.issue.ignore.scanner.IssueExclusionsLoader;
import org.sonar.scanner.mediumtest.AnalysisObservers;
import org.sonar.scanner.postjob.DefaultPostJobContext;
import org.sonar.scanner.postjob.PostJobOptimizer;
import org.sonar.scanner.postjob.PostJobsExecutor;
import org.sonar.scanner.qualitygate.QualityGateCheck;
import org.sonar.scanner.report.ActiveRulesPublisher;
import org.sonar.scanner.report.AnalysisCachePublisher;
import org.sonar.scanner.report.AnalysisContextReportPublisher;
import org.sonar.scanner.report.AnalysisWarningsPublisher;
import org.sonar.scanner.report.CeTaskReportDataHolder;
import org.sonar.scanner.report.ChangedLinesPublisher;
import org.sonar.scanner.report.ComponentsPublisher;
import org.sonar.scanner.report.ContextPropertiesPublisher;
import org.sonar.scanner.report.JavaArchitectureInformationProvider;
import org.sonar.scanner.report.MetadataPublisher;
import org.sonar.scanner.report.ReportPublisher;
import org.sonar.scanner.report.ScannerFileStructureProvider;
import org.sonar.scanner.report.SourcePublisher;
import org.sonar.scanner.report.TestExecutionPublisher;
import org.sonar.scanner.repository.ContextPropertiesCache;
import org.sonar.scanner.repository.DefaultProjectRepositoriesLoader;
import org.sonar.scanner.repository.DefaultQualityProfileLoader;
import org.sonar.scanner.repository.ProjectRepositoriesProvider;
import org.sonar.scanner.repository.QualityProfilesProvider;
import org.sonar.scanner.repository.ReferenceBranchSupplier;
import org.sonar.scanner.repository.language.DefaultLanguagesRepository;
import org.sonar.scanner.repository.settings.DefaultProjectSettingsLoader;
import org.sonar.scanner.rule.ActiveRulesProvider;
import org.sonar.scanner.rule.DefaultActiveRulesLoader;
import org.sonar.scanner.rule.QProfileVerifier;
import org.sonar.scanner.scan.branch.BranchConfiguration;
import org.sonar.scanner.scan.branch.BranchConfigurationProvider;
import org.sonar.scanner.scan.branch.BranchType;
import org.sonar.scanner.scan.branch.ProjectBranchesProvider;
import org.sonar.scanner.scan.filesystem.DefaultProjectFileSystem;
import org.sonar.scanner.scan.filesystem.FileIndexer;
import org.sonar.scanner.scan.filesystem.InputComponentStore;
import org.sonar.scanner.scan.filesystem.LanguageDetection;
import org.sonar.scanner.scan.filesystem.MetadataGenerator;
import org.sonar.scanner.scan.filesystem.ProjectCoverageAndDuplicationExclusions;
import org.sonar.scanner.scan.filesystem.ProjectExclusionFilters;
import org.sonar.scanner.scan.filesystem.ProjectFileIndexer;
import org.sonar.scanner.scan.filesystem.ScannerComponentIdGenerator;
import org.sonar.scanner.scan.filesystem.StatusDetection;
import org.sonar.scanner.scan.measure.DefaultMetricFinder;
import org.sonar.scanner.scm.ScmChangedFilesProvider;
import org.sonar.scanner.scm.ScmConfiguration;
import org.sonar.scanner.scm.ScmPublisher;
import org.sonar.scanner.scm.ScmRevisionImpl;
import org.sonar.scanner.sensor.DefaultSensorStorage;
import org.sonar.scanner.sensor.ExecutingSensorContext;
import org.sonar.scanner.sensor.ProjectSensorContext;
import org.sonar.scanner.sensor.ProjectSensorExtensionDictionary;
import org.sonar.scanner.sensor.ProjectSensorOptimizer;
import org.sonar.scanner.sensor.ProjectSensorsExecutor;
import org.sonar.scanner.sensor.UnchangedFilesHandler;
import org.sonar.scm.git.GitScmSupport;
import org.sonar.scm.svn.SvnScmSupport;

import static org.sonar.api.batch.InstantiationStrategy.PER_BATCH;
import static org.sonar.api.utils.Preconditions.checkNotNull;
import static org.sonar.core.extension.CoreExtensionsInstaller.noExtensionFilter;
import static org.sonar.scanner.bootstrap.ExtensionUtils.isDeprecatedScannerSide;
import static org.sonar.scanner.bootstrap.ExtensionUtils.isInstantiationStrategy;
import static org.sonar.scanner.bootstrap.ExtensionUtils.isScannerSide;
@@ -63,58 +145,182 @@ import static org.sonar.scanner.bootstrap.ExtensionUtils.isScannerSide;
public class SpringProjectScanContainer extends SpringComponentContainer {
private static final Logger LOG = LoggerFactory.getLogger(SpringProjectScanContainer.class);

public SpringProjectScanContainer(SpringComponentContainer parentContainer) {
super(parentContainer);
public SpringProjectScanContainer(SpringComponentContainer globalContainer) {
super(globalContainer);
}

@Override
protected void doBeforeStart() {
Set<String> languages = getParentComponentByType(InputComponentStore.class).languages();
installPluginsForLanguages(languages);
addScannerExtensions();
addScannerComponents();
}

private void installPluginsForLanguages(Set<String> languageKeys) {
ScannerPluginRepository pluginRepository = getParentComponentByType(ScannerPluginRepository.class);
Collection<PluginInfo> languagePlugins = pluginRepository.installPluginsForLanguages(languageKeys);
for (PluginInfo pluginInfo : languagePlugins) {
Plugin instance = pluginRepository.getPluginInstance(pluginInfo.getKey());
addExtension(pluginInfo, instance);
}
getParentComponentByType(ExtensionInstaller.class)
.installExtensionsForPlugins(this, getScannerProjectExtensionsFilter(), languagePlugins);
}

private void addScannerComponents() {
add(

ScanProperties.class,
ProjectReactorBuilder.class,
WorkDirectoriesInitializer.class,
new MutableProjectReactorProvider(),
ProjectBuildersExecutor.class,
ProjectLock.class,
ResourceTypes.class,
ProjectReactorValidator.class,
ProjectInfo.class,
new BranchConfigurationProvider(),
new ProjectBranchesProvider(),
ProjectRepositoriesProvider.class,
new ProjectServerSettingsProvider(),
AnalysisCacheEnabled.class,
DeprecatedPropertiesWarningGenerator.class,

// lang
LanguagesProvider.class,
// temp
new AnalysisTempFolderProvider(),

// file system
ModuleIndexer.class,
InputComponentStore.class,
PathResolver.class,
new InputProjectProvider(),
new InputModuleHierarchyProvider(),
ScannerComponentIdGenerator.class,
new ScmChangedFilesProvider(),
StatusDetection.class,
LanguageDetection.class,
MetadataGenerator.class,
FileMetadata.class,
FileIndexer.class,
ProjectFileIndexer.class,
ProjectExclusionFilters.class,

// rules
new ActiveRulesProvider(),
new QualityProfilesProvider(),
CheckFactory.class,
QProfileVerifier.class,

// issues
DefaultNoSonarFilter.class,
IssueFilters.class,
IssuePublisher.class,

// metrics
DefaultMetricFinder.class,

// lang
LanguagesProvider.class,
DefaultLanguagesRepository.class,

// issue exclusions
IssueInclusionPatternInitializer.class,
IssueExclusionPatternInitializer.class,
IssueExclusionsLoader.class,
EnforceIssuesFilter.class,
IgnoreIssuesFilter.class,

// context
ContextPropertiesCache.class,
ContextPropertiesPublisher.class,

SensorStrategy.class,

MutableProjectSettings.class,
ScannerProperties.class,
SonarGlobalPropertiesFilter.class,
ProjectConfigurationProvider.class,

ProjectCoverageAndDuplicationExclusions.class,

// Plugin cache
AnalysisCacheProvider.class,
AnalysisCacheMemoryStorage.class,
DefaultAnalysisCacheLoader.class,

// Report
ReferenceBranchSupplier.class,
ScannerMetrics.class,
JavaArchitectureInformationProvider.class,
ReportPublisher.class,
ScannerFileStructureProvider.class,
AnalysisContextReportPublisher.class,
MetadataPublisher.class,
ActiveRulesPublisher.class,
ComponentsPublisher.class,
AnalysisCachePublisher.class,
TestExecutionPublisher.class,
SourcePublisher.class,
ChangedLinesPublisher.class,
AnalysisWarningsPublisher.class,

CeTaskReportDataHolder.class,

// QualityGate check
QualityGateCheck.class,

// Cpd
CpdExecutor.class,
CpdSettings.class,
SonarCpdBlockIndex.class,

// PostJobs
PostJobsExecutor.class,
PostJobOptimizer.class,
DefaultPostJobContext.class,
PostJobExtensionDictionary.class,

// SCM
ScmConfiguration.class,
ScmPublisher.class,
ScmRevisionImpl.class,

// Sensors
ProjectSensorExtensionDictionary.class,
DefaultSensorStorage.class,
DefaultFileLinesContextFactory.class,
ProjectSensorContext.class,
ProjectSensorOptimizer.class,
ProjectSensorsExecutor.class,
ExecutingSensorContext.class,
ProjectSensorExtensionDictionary.class,
UnchangedFilesHandler.class,

// Filesystem
DefaultProjectFileSystem.class,

// CI
new CiConfigurationProvider(),
AppVeyor.class,
AwsCodeBuild.class,
AzureDevops.class,
Bamboo.class,
BitbucketPipelines.class,
Bitrise.class,
Buildkite.class,
CircleCi.class,
CirrusCi.class,
DroneCi.class,
GithubActions.class,
CodeMagic.class,
GitlabCi.class,
Jenkins.class,
SemaphoreCi.class,
TravisCi.class,

AnalysisObservers.class);

add(GitScmSupport.getObjects());
add(SvnScmSupport.getObjects());

add(DefaultProjectSettingsLoader.class,
DefaultActiveRulesLoader.class,
DefaultQualityProfileLoader.class,
DefaultProjectRepositoriesLoader.class);
}

private void addScannerExtensions() {
checkNotNull(getParent());
getParent().getComponentByType(CoreExtensionsInstaller.class)
.install(this, noExtensionFilter(), extension -> getScannerProjectExtensionsFilter().accept(extension));
getParent().getComponentByType(ExtensionInstaller.class)
.install(this, getScannerProjectExtensionsFilter());
}

static ExtensionMatcher getScannerProjectExtensionsFilter() {
@@ -128,16 +334,29 @@ public class SpringProjectScanContainer extends SpringComponentContainer {

@Override
protected void doAfterStart() {
getParentComponentByType(ScannerMetrics.class).addPluginMetrics(getComponentsByType(Metrics.class));
getComponentByType(ProjectLock.class).tryLock();
GlobalAnalysisMode analysisMode = getComponentByType(GlobalAnalysisMode.class);
InputModuleHierarchy tree = getComponentByType(InputModuleHierarchy.class);
ScanProperties properties = getComponentByType(ScanProperties.class);
properties.validate();

properties.get("sonar.branch").ifPresent(deprecatedBranch -> {
throw MessageException.of("The 'sonar.branch' parameter is no longer supported. You should stop using it. " +
"Branch analysis is available in Developer Edition and above. See https://www.sonarsource.com/plans-and-pricing/developer/ for more information.");
});

if (getComponentByType(Languages.class).all().length == 0) {
throw new IllegalStateException("No language plugins are installed.");
BranchConfiguration branchConfig = getComponentByType(BranchConfiguration.class);
if (branchConfig.branchType() == BranchType.PULL_REQUEST) {
LOG.info("Pull request {} for merge into {} from {}", branchConfig.pullRequestKey(), pullRequestBaseToDisplayName(branchConfig.targetBranchName()),
branchConfig.branchName());
} else if (branchConfig.branchName() != null) {
LOG.info("Branch name: {}", branchConfig.branchName());
}

getComponentByType(DeprecatedPropertiesWarningGenerator.class).execute();

getComponentByType(ProjectFileIndexer.class).index();

// Log detected languages and their profiles after FS is indexed and languages detected
getComponentByType(QProfileVerifier.class).execute();

@@ -163,6 +382,10 @@ public class SpringProjectScanContainer extends SpringComponentContainer {
}
}

private static String pullRequestBaseToDisplayName(@Nullable String pullRequestBase) {
return pullRequestBase != null ? pullRequestBase : "default branch";
}

private void scanRecursively(InputModuleHierarchy tree, DefaultInputModule module) {
for (DefaultInputModule child : tree.children(module)) {
scanRecursively(tree, child);

+ 5
- 32
sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/ScannerPluginInstallerTest.java View File

@@ -23,8 +23,6 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.jar.Attributes;
@@ -53,36 +51,11 @@ public class ScannerPluginInstallerTest {

@Test
public void download_installed_plugins() throws IOException {
WsTestUtil.mockReader(wsClient, "api/plugins/installed",
new InputStreamReader(getClass().getResourceAsStream("ScannerPluginInstallerTest/installed-plugins-ws.json")));
enqueueDownload("scmgit", "abc");
enqueueDownload("java", "def");

Map<String, ScannerPlugin> result = underTest.installRequiredPlugins();

assertThat(result.keySet()).containsExactlyInAnyOrder("scmgit");
ScannerPlugin gitPlugin = result.get("scmgit");
assertThat(gitPlugin.getKey()).isEqualTo("scmgit");
assertThat(gitPlugin.getInfo().getNonNullJarFile()).exists().isFile();
assertThat(gitPlugin.getUpdatedAt()).isEqualTo(100L);

Map<String, ScannerPlugin> result2 = underTest.installPluginsForLanguages(new HashSet<>(List.of("java")));

assertThat(result2.keySet()).containsExactlyInAnyOrder("java");
ScannerPlugin javaPlugin = result2.get("java");
assertThat(javaPlugin.getKey()).isEqualTo("java");
assertThat(javaPlugin.getInfo().getNonNullJarFile()).exists().isFile();
assertThat(javaPlugin.getUpdatedAt()).isEqualTo(200L);
}

@Test
public void download_all_plugins() throws IOException {
WsTestUtil.mockReader(wsClient, "api/plugins/installed",
new InputStreamReader(getClass().getResourceAsStream("ScannerPluginInstallerTest/installed-plugins-ws.json")));
WsTestUtil.mockReader(wsClient, "api/plugins/installed", new InputStreamReader(getClass().getResourceAsStream("ScannerPluginInstallerTest/installed-plugins-ws.json")));
enqueueDownload("scmgit", "abc");
enqueueDownload("java", "def");

Map<String, ScannerPlugin> result = underTest.installAllPlugins();
Map<String, ScannerPlugin> result = underTest.installRemotes();

assertThat(result.keySet()).containsExactlyInAnyOrder("scmgit", "java");
ScannerPlugin gitPlugin = result.get("scmgit");
@@ -100,7 +73,7 @@ public class ScannerPluginInstallerTest {
public void fail_if_json_of_installed_plugins_is_not_valid() {
WsTestUtil.mockReader(wsClient, "api/plugins/installed", new StringReader("not json"));

assertThatThrownBy(() -> underTest.installRequiredPlugins())
assertThatThrownBy(() -> underTest.installRemotes())
.isInstanceOf(IllegalStateException.class)
.hasMessage("Fail to parse response of api/plugins/installed");
}
@@ -114,7 +87,7 @@ public class ScannerPluginInstallerTest {
enqueueDownload("java", "def");
enqueueDownload("cobol", "ghi");

Map<String, ScannerPlugin> result = underTest.installRequiredPlugins();
Map<String, ScannerPlugin> result = underTest.installRemotes();

assertThat(result.keySet()).containsExactlyInAnyOrder("java", "cobol");
}
@@ -128,7 +101,7 @@ public class ScannerPluginInstallerTest {
enqueueDownload("cobol", "ghi");
enqueueNotFoundDownload("java", "def");

assertThatThrownBy(() -> underTest.installRequiredPlugins())
assertThatThrownBy(() -> underTest.installRemotes())
.isInstanceOf(IllegalStateException.class)
.hasMessage("Fail to download plugin [java]. Not found.");
}

+ 13
- 50
sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/ScannerPluginRepositoryTest.java View File

@@ -21,16 +21,9 @@ package org.sonar.scanner.bootstrap;

import com.google.common.collect.ImmutableMap;
import java.io.File;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import org.junit.Test;
import org.sonar.api.Plugin;
import org.sonar.api.config.Configuration;
import org.sonar.core.platform.ExplodedPlugin;
import org.sonar.core.platform.PluginClassLoader;
import org.sonar.core.platform.PluginInfo;
@@ -39,12 +32,9 @@ import org.sonar.core.plugin.PluginType;

import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.fail;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyCollection;
import static org.mockito.ArgumentMatchers.anyMap;
import static org.mockito.ArgumentMatchers.anySet;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

@@ -53,57 +43,30 @@ public class ScannerPluginRepositoryTest {
PluginInstaller installer = mock(PluginInstaller.class);
PluginClassLoader loader = mock(PluginClassLoader.class);
PluginJarExploder exploder = new FakePluginJarExploder();
Configuration properties = mock(Configuration.class);
ScannerPluginRepository underTest = new ScannerPluginRepository(installer, exploder, loader, properties);
ScannerPluginRepository underTest = new ScannerPluginRepository(installer, exploder, loader);

@Test
public void install_and_load_plugins() {
PluginInfo squidInfo = new PluginInfo("squid");
PluginInfo javaInfo = new PluginInfo("java");
Map<String, ScannerPlugin> globalPlugins = Map.of("squid", new ScannerPlugin("squid", 1L, PluginType.BUNDLED, squidInfo));
Map<String, ScannerPlugin> languagePlugins = Map.of("java", new ScannerPlugin("java", 1L, PluginType.EXTERNAL, javaInfo));
Plugin squidInstance = mock(Plugin.class);
Plugin javaInstance = mock(Plugin.class);
when(loader.load(anyMap()))
.thenReturn(ImmutableMap.of("squid", squidInstance))
.thenReturn(ImmutableMap.of("java", javaInstance));

when(installer.installRequiredPlugins()).thenReturn(globalPlugins);
when(installer.installPluginsForLanguages(anySet())).thenReturn(languagePlugins);
PluginInfo info = new PluginInfo("java");
ImmutableMap<String, ScannerPlugin> plugins = ImmutableMap.of("java", new ScannerPlugin("java", 1L, PluginType.EXTERNAL, info));
Plugin instance = mock(Plugin.class);
when(loader.load(anyMap())).thenReturn(ImmutableMap.of("java", instance));
when(installer.installRemotes()).thenReturn(plugins);

underTest.start();

assertThat(underTest.getPluginInfos()).containsOnly(squidInfo);
assertThat(underTest.getPluginsByKey()).isEqualTo(globalPlugins);
assertThat(underTest.getPluginInfo("squid")).isSameAs(squidInfo);
assertThat(underTest.getPluginInstance("squid")).isSameAs(squidInstance);

Collection<PluginInfo> result = underTest.installPluginsForLanguages(new HashSet<>(List.of("java")));

assertThat(result).containsOnly(javaInfo);
assertThat(underTest.getPluginInfos()).containsExactlyInAnyOrder(squidInfo, javaInfo);
assertThat(underTest.getExternalPluginsInfos()).containsExactlyInAnyOrder(javaInfo);
assertThat(underTest.getPluginsByKey().values()).containsExactlyInAnyOrder(globalPlugins.get("squid"), languagePlugins.get("java"));
assertThat(underTest.getPluginInfos()).containsOnly(info);
assertThat(underTest.getPluginsByKey()).isEqualTo(plugins);
assertThat(underTest.getPluginInfo("java")).isSameAs(info);
assertThat(underTest.getPluginInstance("java")).isSameAs(instance);
assertThat(underTest.getPluginInstances()).containsOnly(instance);
assertThat(underTest.getBundledPluginsInfos()).isEmpty();
assertThat(underTest.getExternalPluginsInfos()).isEqualTo(underTest.getPluginInfos());

underTest.stop();
verify(loader).unload(anyCollection());
}

@Test
public void should_install_all_plugins_when_loadall_flag_is_set() {
when(properties.getBoolean("sonar.plugins.loadAll")).thenReturn(Optional.of(true));

underTest.start();

verify(installer).installAllPlugins();
verify(installer, never()).installRequiredPlugins();

Collection<PluginInfo> result = underTest.installPluginsForLanguages(Set.of("java"));

assertThat(result).isEmpty();
verify(installer, never()).installPluginsForLanguages(any());
}

@Test
public void fail_if_requesting_missing_plugin() {
underTest.start();

+ 68
- 96
sonar-scanner-engine/src/test/java/org/sonar/scanner/repository/language/DefaultLanguagesRepositoryTest.java View File

@@ -19,128 +19,100 @@
*/
package org.sonar.scanner.repository.language;

import java.io.InputStreamReader;
import java.io.StringReader;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.slf4j.event.Level;
import org.sonar.api.config.Configuration;
import org.sonar.api.resources.Language;
import org.sonar.api.resources.Languages;
import org.sonar.api.testfixtures.log.LogTester;
import org.sonar.scanner.WsTestUtil;
import org.sonar.scanner.bootstrap.DefaultScannerWsClient;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.AssertionsForClassTypes.catchThrowableOfType;
import static org.assertj.core.api.Assertions.tuple;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

public class DefaultLanguagesRepositoryTest {

@Rule
public LogTester logTester = new LogTester();

private final DefaultScannerWsClient wsClient = mock(DefaultScannerWsClient.class);
private final Configuration properties = mock(Configuration.class);
private final Languages languages = mock(Languages.class);
private final DefaultLanguagesRepository underTest = new DefaultLanguagesRepository(wsClient, properties);

private static final String[] JAVA_SUFFIXES = new String[] { ".java", ".jav" };
private static final String[] XOO_SUFFIXES = new String[] { ".xoo" };
private static final String[] XOO_PATTERNS = new String[] { "Xoofile" };
private static final String[] PYTHON_SUFFIXES = new String[] { ".py" };

@Before
public void setup() {
logTester.setLevel(Level.DEBUG);
}

@Test
public void should_load_languages_from_ws() {
WsTestUtil.mockReader(wsClient, "/api/languages/list",
new InputStreamReader(getClass().getResourceAsStream("DefaultLanguageRepositoryTest/languages-ws.json")));

when(properties.getStringArray("sonar.java.file.suffixes")).thenReturn(JAVA_SUFFIXES);
when(properties.getStringArray("sonar.xoo.file.suffixes")).thenReturn(XOO_SUFFIXES);
when(properties.getStringArray("sonar.python.file.suffixes")).thenReturn(PYTHON_SUFFIXES);

underTest.start();
underTest.stop();

assertThat(underTest.all()).hasSize(3);
assertThat(underTest.get("java")).isNotNull();
assertThat(underTest.get("java").fileSuffixes()).containsExactlyInAnyOrder(JAVA_SUFFIXES);
assertThat(underTest.get("java").isPublishAllFiles()).isTrue();
assertThat(underTest.get("xoo")).isNotNull();
assertThat(underTest.get("xoo").fileSuffixes()).containsExactlyInAnyOrder(XOO_SUFFIXES);
assertThat(underTest.get("xoo").isPublishAllFiles()).isTrue();
assertThat(underTest.get("python")).isNotNull();
assertThat(underTest.get("python").fileSuffixes()).containsExactlyInAnyOrder(PYTHON_SUFFIXES);
assertThat(underTest.get("python").isPublishAllFiles()).isTrue();
}

@Test
public void should_throw_error_on_invalid_ws_response() {
WsTestUtil.mockReader(wsClient, "api/languages/list", new StringReader("not json"));

IllegalStateException e = catchThrowableOfType(underTest::start, IllegalStateException.class);

assertThat(e).hasMessage("Fail to parse response of /api/languages/list");
}
private final DefaultLanguagesRepository underTest = new DefaultLanguagesRepository(languages);

@Test
public void should_return_null_when_language_not_found() {
WsTestUtil.mockReader(wsClient, "/api/languages/list",
new InputStreamReader(getClass().getResourceAsStream("DefaultLanguageRepositoryTest/languages-ws.json")));

when(properties.getStringArray("sonar.java.file.suffixes")).thenReturn(JAVA_SUFFIXES);
when(properties.getStringArray("sonar.xoo.file.suffixes")).thenReturn(XOO_SUFFIXES);
when(properties.getStringArray("sonar.python.file.suffixes")).thenReturn(PYTHON_SUFFIXES);

assertThat(underTest.get("k1")).isNull();
public void returns_all_languages() {
when(languages.all()).thenReturn(new Language[] {new TestLanguage("k1", true), new TestLanguage("k2", false)});
assertThat(underTest.all())
.extracting("key", "name", "fileSuffixes", "publishAllFiles")
.containsOnly(
tuple("k1", "name k1", new String[] {"k1"}, true),
tuple("k2", "name k2", new String[] {"k2"}, false)
);
}

@Test
public void publishAllFiles_by_default() {
WsTestUtil.mockReader(wsClient, "/api/languages/list",
new InputStreamReader(getClass().getResourceAsStream("DefaultLanguageRepositoryTest/languages-ws.json")));

underTest.start();
underTest.stop();

assertThat(underTest.get("java").isPublishAllFiles()).isTrue();
assertThat(underTest.get("xoo").isPublishAllFiles()).isTrue();
assertThat(underTest.get("python").isPublishAllFiles()).isTrue();
when(languages.all()).thenReturn(new Language[] {new TestLanguage2("k1"), new TestLanguage2("k2")});
assertThat(underTest.all())
.extracting("key", "name", "fileSuffixes", "publishAllFiles")
.containsOnly(
tuple("k1", "name k1", new String[] {"k1"}, true),
tuple("k2", "name k2", new String[] {"k2"}, true)
);
}

@Test
public void get_find_language_by_key() {
WsTestUtil.mockReader(wsClient, "/api/languages/list",
new InputStreamReader(getClass().getResourceAsStream("DefaultLanguageRepositoryTest/languages-ws.json")));

when(properties.getStringArray("sonar.java.file.suffixes")).thenReturn(JAVA_SUFFIXES);

underTest.start();
underTest.stop();

assertThat(underTest.get("java"))
when(languages.get("k1")).thenReturn(new TestLanguage2("k1"));
assertThat(underTest.get("k1"))
.extracting("key", "name", "fileSuffixes", "publishAllFiles")
.containsOnly("java", "Java", JAVA_SUFFIXES, true);
.containsOnly("k1", "name k1", new String[] {"k1"}, true);
}

private static class TestLanguage implements Language {
private final String key;
private final boolean publishAllFiles;

public TestLanguage(String key, boolean publishAllFiles) {
this.key = key;
this.publishAllFiles = publishAllFiles;
}

@Override
public String getKey() {
return key;
}

@Override
public String getName() {
return "name " + key;
}

@Override
public String[] getFileSuffixes() {
return new String[] {key};
}

@Override
public boolean publishAllFiles() {
return publishAllFiles;
}
}

@Test
public void should_log_if_language_has_no_suffixes_or_patterns() {
WsTestUtil.mockReader(wsClient, "/api/languages/list",
new InputStreamReader(getClass().getResourceAsStream("DefaultLanguageRepositoryTest/languages-ws.json")));
private static class TestLanguage2 implements Language {
private final String key;

public TestLanguage2(String key) {
this.key = key;
}

when(properties.getStringArray("sonar.java.file.suffixes")).thenReturn(JAVA_SUFFIXES);
when(properties.getStringArray("sonar.xoo.file.patterns")).thenReturn(XOO_PATTERNS);
@Override
public String getKey() {
return key;
}

underTest.start();
@Override
public String getName() {
return "name " + key;
}

assertThat(logTester.logs(Level.DEBUG)).contains("Language 'Python' cannot be detected as it has neither suffixes nor patterns.");
@Override
public String[] getFileSuffixes() {
return new String[] {key};
}
}

}

+ 0
- 27
sonar-scanner-engine/src/test/java/org/sonar/scanner/repository/settings/AbstractSettingsLoaderTest.java View File

@@ -89,31 +89,4 @@ public class AbstractSettingsLoaderTest {
entry("sonar.issue.exclusions.multicriteria.2.rulepattern", "*:S456"));
}

@Test
public void should_always_load_language_detection_properties() {
assertThat(AbstractSettingsLoader.toMap(List.of(
Setting.newBuilder()
.setInherited(false)
.setKey("sonar.xoo.file.suffixes")
.setValues(Values.newBuilder().addValues(".xoo")).build(),
Setting.newBuilder()
.setInherited(false)
.setKey("sonar.xoo.file.patterns")
.setValues(Values.newBuilder().addValues("Xoofile")).build()
))).containsExactly(
entry("sonar.xoo.file.suffixes", ".xoo"),
entry("sonar.xoo.file.patterns", "Xoofile")
);
}

@Test
public void should_not_load_inherited_properties() {
assertThat(AbstractSettingsLoader.toMap(List.of(
Setting.newBuilder()
.setInherited(true)
.setKey("sonar.inherited.property")
.setValues(Values.newBuilder().addValues("foo")).build()
))).isEmpty();
}

}

+ 8
- 8
sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java View File

@@ -33,7 +33,7 @@ import org.sonar.api.config.internal.MapSettings;
import org.sonar.api.resources.Language;
import org.sonar.api.resources.Languages;
import org.sonar.api.utils.MessageException;
import org.sonar.scanner.mediumtest.FakeLanguagesRepository;
import org.sonar.scanner.repository.language.DefaultLanguagesRepository;
import org.sonar.scanner.repository.language.LanguagesRepository;

import static org.assertj.core.api.Assertions.assertThat;
@@ -71,7 +71,7 @@ public class LanguageDetectionTest {

@Test
public void detectLanguageKey_shouldDetectByFileExtension() {
LanguagesRepository languages = new FakeLanguagesRepository(new Languages(new MockLanguage("java", "java", "jav"), new MockLanguage("cobol", "cbl", "cob")));
LanguagesRepository languages = new DefaultLanguagesRepository(new Languages(new MockLanguage("java", "java", "jav"), new MockLanguage("cobol", "cbl", "cob")));
LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);

assertThat(detectLanguageKey(detection, "Foo.java")).isEqualTo("java");
@@ -90,7 +90,7 @@ public class LanguageDetectionTest {
@Test
@UseDataProvider("filenamePatterns")
public void detectLanguageKey_shouldDetectByFileNamePattern(String fileName, String expectedLanguageKey) {
LanguagesRepository languages = new FakeLanguagesRepository(new Languages(
LanguagesRepository languages = new DefaultLanguagesRepository(new Languages(
new MockLanguage("docker", new String[0], new String[] {"*.dockerfile", "*.Dockerfile", "Dockerfile", "Dockerfile.*"}),
new MockLanguage("terraform", new String[] {"tf"}, new String[] {".tf"}),
new MockLanguage("java", new String[0], new String[] {"**/*Test.java"})));
@@ -117,13 +117,13 @@ public class LanguageDetectionTest {

@Test
public void detectLanguageKey_shouldNotFailIfNoLanguage() {
LanguageDetection detection = spy(new LanguageDetection(settings.asConfig(), new FakeLanguagesRepository(new Languages())));
LanguageDetection detection = spy(new LanguageDetection(settings.asConfig(), new DefaultLanguagesRepository(new Languages())));
assertThat(detectLanguageKey(detection, "Foo.java")).isNull();
}

@Test
public void detectLanguageKey_shouldAllowPluginsToDeclareFileExtensionTwiceForCaseSensitivity() {
LanguagesRepository languages = new FakeLanguagesRepository(new Languages(new MockLanguage("abap", "abap", "ABAP")));
LanguagesRepository languages = new DefaultLanguagesRepository(new Languages(new MockLanguage("abap", "abap", "ABAP")));

LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
assertThat(detectLanguageKey(detection, "abc.abap")).isEqualTo("abap");
@@ -131,7 +131,7 @@ public class LanguageDetectionTest {

@Test
public void detectLanguageKey_shouldFailIfConflictingLanguageSuffix() {
LanguagesRepository languages = new FakeLanguagesRepository(new Languages(new MockLanguage("xml", "xhtml"), new MockLanguage("web", "xhtml")));
LanguagesRepository languages = new DefaultLanguagesRepository(new Languages(new MockLanguage("xml", "xhtml"), new MockLanguage("web", "xhtml")));
LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
assertThatThrownBy(() -> detectLanguageKey(detection, "abc.xhtml"))
.isInstanceOf(MessageException.class)
@@ -142,7 +142,7 @@ public class LanguageDetectionTest {

@Test
public void detectLanguageKey_shouldSolveConflictUsingFilePattern() {
LanguagesRepository languages = new FakeLanguagesRepository(new Languages(new MockLanguage("xml", "xhtml"), new MockLanguage("web", "xhtml")));
LanguagesRepository languages = new DefaultLanguagesRepository(new Languages(new MockLanguage("xml", "xhtml"), new MockLanguage("web", "xhtml")));

settings.setProperty("sonar.lang.patterns.xml", "xml/**");
settings.setProperty("sonar.lang.patterns.web", "web/**");
@@ -153,7 +153,7 @@ public class LanguageDetectionTest {

@Test
public void detectLanguageKey_shouldFailIfConflictingFilePattern() {
LanguagesRepository languages = new FakeLanguagesRepository(new Languages(new MockLanguage("abap", "abap"), new MockLanguage("cobol", "cobol")));
LanguagesRepository languages = new DefaultLanguagesRepository(new Languages(new MockLanguage("abap", "abap"), new MockLanguage("cobol", "cobol")));
settings.setProperty("sonar.lang.patterns.abap", "*.abap,*.txt");
settings.setProperty("sonar.lang.patterns.cobol", "*.cobol,*.txt");


+ 2
- 6
sonar-scanner-engine/src/test/resources/org/sonar/scanner/bootstrap/ScannerPluginInstallerTest/installed-plugins-ws.json View File

@@ -4,17 +4,13 @@
"key": "scmgit",
"hash": "abc",
"type": "BUNDLED",
"updatedAt": 100,
"requiredForLanguages": []
"updatedAt": 100
},
{
"key": "java",
"hash": "def",
"type": "EXTERNAL",
"updatedAt": 200,
"requiredForLanguages": [
"java"
]
"updatedAt": 200
}
]
}

+ 0
- 16
sonar-scanner-engine/src/test/resources/org/sonar/scanner/repository/language/DefaultLanguageRepositoryTest/languages-ws.json View File

@@ -1,16 +0,0 @@
{
"languages": [
{
"key": "java",
"name": "Java"
},
{
"key" : "xoo",
"name": "Xoo"
},
{
"key": "python",
"name": "Python"
}
]
}

Loading…
Cancel
Save