From 7013e543f07fca1831fd1efee29997981b6ec19b Mon Sep 17 00:00:00 2001 From: Matteo Mara Date: Wed, 6 Dec 2023 16:58:03 +0100 Subject: SONAR-21195 Enhance scanner engine to download only required plugins --- .../scanner/bootstrap/ExtensionInstaller.java | 11 +- .../sonar/scanner/bootstrap/PluginInstaller.java | 17 +- .../scanner/bootstrap/ScannerPluginInstaller.java | 83 ++++- .../scanner/bootstrap/ScannerPluginRepository.java | 58 +++- .../scanner/bootstrap/SpringGlobalContainer.java | 9 +- .../scanner/bootstrap/SpringScannerContainer.java | 348 +++++++++++++++++++++ .../mediumtest/FakeLanguagesRepository.java | 83 +++++ .../scanner/mediumtest/FakePluginInstaller.java | 13 +- .../language/DefaultLanguagesRepository.java | 70 ++++- .../repository/language/SupportedLanguageDto.java | 63 ++++ .../settings/AbstractSettingsLoader.java | 4 +- .../scanner/scan/SpringProjectScanContainer.java | 284 ++--------------- 12 files changed, 742 insertions(+), 301 deletions(-) create mode 100644 sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java create mode 100644 sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakeLanguagesRepository.java create mode 100644 sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/SupportedLanguageDto.java (limited to 'sonar-scanner-engine/src/main/java') diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ExtensionInstaller.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ExtensionInstaller.java index 5df9f8481fc..d9561394360 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ExtensionInstaller.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ExtensionInstaller.java @@ -19,6 +19,7 @@ */ package org.sonar.scanner.bootstrap; +import java.util.Collection; import javax.annotation.Nullable; import org.sonar.api.Plugin; import org.sonar.api.SonarRuntime; @@ -47,7 +48,13 @@ public class ExtensionInstaller { } // plugin extensions - for (PluginInfo pluginInfo : pluginRepository.getPluginInfos()) { + installExtensionsForPlugins(container, matcher, pluginRepository.getPluginInfos()); + + return this; + } + + public void installExtensionsForPlugins(ExtensionContainer container, ExtensionMatcher matcher, Collection pluginInfos) { + for (PluginInfo pluginInfo : pluginInfos) { Plugin plugin = pluginRepository.getPluginInstance(pluginInfo.getKey()); Plugin.Context context = new PluginContextImpl.Builder() .setSonarRuntime(sonarRuntime) @@ -59,8 +66,6 @@ public class ExtensionInstaller { doInstall(container, matcher, pluginInfo, extension); } } - - return this; } private static void doInstall(ExtensionContainer container, ExtensionMatcher matcher, @Nullable PluginInfo pluginInfo, Object extension) { diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java index 9b2c10a25f9..0e117e75136 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java @@ -21,15 +21,28 @@ package org.sonar.scanner.bootstrap; import java.util.List; import java.util.Map; +import java.util.Set; public interface PluginInstaller { /** - * Gets the list of plugins installed on server and downloads them if not + * Loads/downloads all plugins that are installed on the server. + * @return information about all installed plugins, grouped by key + */ + Map installAllPlugins(); + + /** + * Gets the list of plugins that are not required for any specific languages and downloads them if not * already in local cache. * @return information about all installed plugins, grouped by key */ - Map installRemotes(); + Map installRequiredPlugins(); + + /** + * Loads/downloads plugins that are required for the given languageKeys. + * @return information about any plugins installed by this call, grouped by key + */ + Map installPluginsForLanguages(Set languageKeys); /** * Used only by medium tests. diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java index 1b22fa6397a..d2453c3b6ec 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java @@ -22,11 +22,14 @@ package org.sonar.scanner.bootstrap; import com.google.gson.Gson; import java.io.File; import java.io.Reader; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.function.Predicate; import javax.annotation.Nullable; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; @@ -48,21 +51,58 @@ public class ScannerPluginInstaller implements PluginInstaller { private final PluginFiles pluginFiles; private final DefaultScannerWsClient wsClient; + private List availablePlugins; + public ScannerPluginInstaller(PluginFiles pluginFiles, DefaultScannerWsClient wsClient) { this.pluginFiles = pluginFiles; this.wsClient = wsClient; } @Override - public Map installRemotes() { + public Map installAllPlugins() { + LOG.info("Loading all plugins"); + return installPlugins(p -> true).installedPluginsByKey; + } + + @Override + public Map installRequiredPlugins() { + LOG.info("Loading required plugins"); + InstallResult result = installPlugins(p -> p.getRequiredForLanguages() == null || p.getRequiredForLanguages().isEmpty()); + + LOG.debug("Plugins not loaded because they are optional: {}", result.skippedPlugins); + + return result.installedPluginsByKey; + } + + @Override + public Map installPluginsForLanguages(Set languageKeys) { + LOG.info("Loading plugins for detected languages"); + LOG.debug("Detected languages: {}", languageKeys); + InstallResult result = installPlugins( + p -> p.getRequiredForLanguages() != null && !Collections.disjoint(p.getRequiredForLanguages(), languageKeys) + ); + + List skippedLanguagePlugins = result.skippedPlugins.stream() + .filter(p -> p.getRequiredForLanguages() != null && !p.getRequiredForLanguages().isEmpty()).toList(); + LOG.debug("Optional language-specific plugins not loaded: {}", skippedLanguagePlugins); + + return result.installedPluginsByKey; + } + + private InstallResult installPlugins(Predicate pluginFilter) { + if (this.availablePlugins == null) { + this.availablePlugins = listInstalledPlugins(); + } + Profiler profiler = Profiler.create(LOG).startInfo("Load/download plugins"); try { - Map result = new HashMap<>(); - Loaded loaded = loadPlugins(result); + InstallResult result = new InstallResult(); + Loaded loaded = loadPlugins(result, pluginFilter); if (!loaded.ok) { // retry once, a plugin may have been uninstalled during downloads - result.clear(); - loaded = loadPlugins(result); + this.availablePlugins = listInstalledPlugins(); + result.installedPluginsByKey.clear(); + loaded = loadPlugins(result, pluginFilter); if (!loaded.ok) { throw new IllegalStateException(format("Fail to download plugin [%s]. Not found.", loaded.notFoundPlugin)); } @@ -73,16 +113,23 @@ public class ScannerPluginInstaller implements PluginInstaller { } } - private Loaded loadPlugins(Map result) { - for (InstalledPlugin plugin : listInstalledPlugins()) { + private Loaded loadPlugins(InstallResult result, Predicate pluginFilter) { + List pluginsToInstall = availablePlugins.stream() + .filter(pluginFilter).toList(); + + for (InstalledPlugin plugin : pluginsToInstall) { Optional jarFile = pluginFiles.get(plugin); if (jarFile.isEmpty()) { return new Loaded(false, plugin.key); } PluginInfo info = PluginInfo.create(jarFile.get()); - result.put(info.getKey(), new ScannerPlugin(plugin.key, plugin.updatedAt, PluginType.valueOf(plugin.type), info)); + result.installedPluginsByKey.put(info.getKey(), new ScannerPlugin(plugin.key, plugin.updatedAt, PluginType.valueOf(plugin.type), info)); } + + result.skippedPlugins = availablePlugins.stream() + .filter(Predicate.not(pluginFilter)).toList(); + return new Loaded(true, null); } @@ -97,7 +144,7 @@ public class ScannerPluginInstaller implements PluginInstaller { /** * Gets information about the plugins installed on server (filename, checksum) */ - private InstalledPlugin[] listInstalledPlugins() { + private List listInstalledPlugins() { Profiler profiler = Profiler.create(LOG).startInfo("Load plugins index"); GetRequest getRequest = new GetRequest(PLUGINS_WS_URL); InstalledPlugins installedPlugins; @@ -111,8 +158,13 @@ public class ScannerPluginInstaller implements PluginInstaller { return installedPlugins.plugins; } + private static class InstallResult { + Map installedPluginsByKey = new HashMap<>(); + List skippedPlugins = new ArrayList<>(); + } + private static class InstalledPlugins { - InstalledPlugin[] plugins; + List plugins; public InstalledPlugins() { // http://stackoverflow.com/a/18645370/229031 @@ -124,10 +176,21 @@ public class ScannerPluginInstaller implements PluginInstaller { String hash; long updatedAt; String type; + private Set requiredForLanguages; public InstalledPlugin() { // http://stackoverflow.com/a/18645370/229031 } + + public Set getRequiredForLanguages() { + return requiredForLanguages; + } + + @Override + public String toString() { + return key; + } + } private static class Loaded { diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java index 1f8c747678b..4d1894b59f8 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java @@ -20,14 +20,16 @@ package org.sonar.scanner.bootstrap; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.Map; -import java.util.stream.Collectors; +import java.util.Set; import javax.annotation.CheckForNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.api.Plugin; import org.sonar.api.Startable; +import org.sonar.api.config.Configuration; import org.sonar.core.platform.ExplodedPlugin; import org.sonar.core.platform.PluginClassLoader; import org.sonar.core.platform.PluginInfo; @@ -35,6 +37,7 @@ import org.sonar.core.platform.PluginJarExploder; import org.sonar.core.platform.PluginRepository; import org.sonar.core.plugin.PluginType; +import static java.util.stream.Collectors.toMap; import static org.sonar.api.utils.Preconditions.checkState; /** @@ -47,22 +50,33 @@ public class ScannerPluginRepository implements PluginRepository, Startable { private final PluginJarExploder pluginJarExploder; private final PluginClassLoader loader; + private final Configuration properties; + private Map pluginInstancesByKeys; private Map pluginsByKeys; private Map keysByClassLoader; + private boolean shouldLoadAllPluginsOnStart; - public ScannerPluginRepository(PluginInstaller installer, PluginJarExploder pluginJarExploder, PluginClassLoader loader) { + public ScannerPluginRepository(PluginInstaller installer, PluginJarExploder pluginJarExploder, PluginClassLoader loader, Configuration properties) { this.installer = installer; this.pluginJarExploder = pluginJarExploder; this.loader = loader; + this.properties = properties; } @Override public void start() { - pluginsByKeys = new HashMap<>(installer.installRemotes()); - Map explodedPLuginsByKey = pluginsByKeys.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> pluginJarExploder.explode(e.getValue().getInfo()))); - pluginInstancesByKeys = new HashMap<>(loader.load(explodedPLuginsByKey)); + shouldLoadAllPluginsOnStart = properties.getBoolean("sonar.plugins.loadAll").orElse(false); + if (shouldLoadAllPluginsOnStart) { + LOG.warn("sonar.plugins.loadAll is true, so ALL available plugins will be downloaded"); + pluginsByKeys = new HashMap<>(installer.installAllPlugins()); + } else { + pluginsByKeys = new HashMap<>(installer.installRequiredPlugins()); + } + + Map explodedPluginsByKey = pluginsByKeys.entrySet().stream() + .collect(toMap(Map.Entry::getKey, e -> pluginJarExploder.explode(e.getValue().getInfo()))); + pluginInstancesByKeys = new HashMap<>(loader.load(explodedPluginsByKey)); // this part is only used by medium tests for (Object[] localPlugin : installer.installLocals()) { @@ -77,7 +91,29 @@ public class ScannerPluginRepository implements PluginRepository, Startable { keysByClassLoader.put(e.getValue().getClass().getClassLoader(), e.getKey()); } - logPlugins(); + logPlugins(pluginsByKeys.values()); + } + + public Collection installPluginsForLanguages(Set languageKeys) { + if (shouldLoadAllPluginsOnStart) { + return Collections.emptySet(); + } + + var languagePluginsByKeys = new HashMap<>(installer.installPluginsForLanguages(languageKeys)); + + pluginsByKeys.putAll(languagePluginsByKeys); + + Map explodedPluginsByKey = languagePluginsByKeys.entrySet().stream() + .collect(toMap(Map.Entry::getKey, e -> pluginJarExploder.explode(e.getValue().getInfo()))); + pluginInstancesByKeys.putAll(new HashMap<>(loader.load(explodedPluginsByKey))); + + keysByClassLoader = new HashMap<>(); + for (Map.Entry e : pluginInstancesByKeys.entrySet()) { + keysByClassLoader.put(e.getValue().getClass().getClassLoader(), e.getKey()); + } + + logPlugins(languagePluginsByKeys.values()); + return languagePluginsByKeys.values().stream().map(ScannerPlugin::getInfo).toList(); } @CheckForNull @@ -85,12 +121,12 @@ public class ScannerPluginRepository implements PluginRepository, Startable { return keysByClassLoader.get(cl); } - private void logPlugins() { - if (pluginsByKeys.isEmpty()) { + private static void logPlugins(Collection plugins) { + if (plugins.isEmpty()) { LOG.debug("No plugins loaded"); } else { - LOG.debug("Plugins:"); - for (ScannerPlugin p : pluginsByKeys.values()) { + LOG.debug("Plugins loaded:"); + for (ScannerPlugin p : plugins) { LOG.debug(" * {} {} ({})", p.getName(), p.getVersion(), p.getKey()); } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringGlobalContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringGlobalContainer.java index f0e1983a8c3..3c3bf87af47 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringGlobalContainer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringGlobalContainer.java @@ -54,9 +54,8 @@ import org.sonar.scanner.repository.DefaultMetricsRepositoryLoader; import org.sonar.scanner.repository.DefaultNewCodePeriodLoader; import org.sonar.scanner.repository.MetricsRepositoryProvider; import org.sonar.scanner.repository.settings.DefaultGlobalSettingsLoader; -import org.sonar.scanner.scan.SpringProjectScanContainer; -@Priority(3) +@Priority(4) public class SpringGlobalContainer extends SpringComponentContainer { private static final Logger LOG = LoggerFactory.getLogger(SpringGlobalContainer.class); private final Map scannerProperties; @@ -120,7 +119,7 @@ public class SpringGlobalContainer extends SpringComponentContainer { @Override protected void doAfterStart() { - installPlugins(); + installRequiredPlugins(); loadCoreExtensions(); long startTime = System.currentTimeMillis(); @@ -136,12 +135,12 @@ public class SpringGlobalContainer extends SpringComponentContainer { throw MessageException.of("The preview mode, along with the 'sonar.analysis.mode' parameter, is no more supported. You should stop using this parameter."); } getComponentByType(RuntimeJavaVersion.class).checkJavaVersion(); - new SpringProjectScanContainer(this).execute(); + new SpringScannerContainer(this).execute(); LOG.info("Analysis total time: {}", formatTime(System.currentTimeMillis() - startTime)); } - private void installPlugins() { + private void installRequiredPlugins() { PluginRepository pluginRepository = getComponentByType(PluginRepository.class); for (PluginInfo pluginInfo : pluginRepository.getPluginInfos()) { Plugin instance = pluginRepository.getPluginInstance(pluginInfo.getKey()); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java new file mode 100644 index 00000000000..0cba8e33dfc --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java @@ -0,0 +1,348 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.bootstrap; + +import javax.annotation.Nullable; +import javax.annotation.Priority; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.batch.fs.internal.FileMetadata; +import org.sonar.api.batch.fs.internal.SensorStrategy; +import org.sonar.api.batch.rule.CheckFactory; +import org.sonar.api.batch.sensor.issue.internal.DefaultNoSonarFilter; +import org.sonar.api.scan.filesystem.PathResolver; +import org.sonar.api.utils.MessageException; +import org.sonar.core.extension.CoreExtensionsInstaller; +import org.sonar.core.metric.ScannerMetrics; +import org.sonar.core.platform.SpringComponentContainer; +import org.sonar.scanner.DefaultFileLinesContextFactory; +import org.sonar.scanner.ProjectInfo; +import org.sonar.scanner.analysis.AnalysisTempFolderProvider; +import org.sonar.scanner.cache.AnalysisCacheEnabled; +import org.sonar.scanner.cache.AnalysisCacheMemoryStorage; +import org.sonar.scanner.cache.AnalysisCacheProvider; +import org.sonar.scanner.cache.DefaultAnalysisCacheLoader; +import org.sonar.scanner.ci.CiConfigurationProvider; +import org.sonar.scanner.ci.vendors.AppVeyor; +import org.sonar.scanner.ci.vendors.AwsCodeBuild; +import org.sonar.scanner.ci.vendors.AzureDevops; +import org.sonar.scanner.ci.vendors.Bamboo; +import org.sonar.scanner.ci.vendors.BitbucketPipelines; +import org.sonar.scanner.ci.vendors.Bitrise; +import org.sonar.scanner.ci.vendors.Buildkite; +import org.sonar.scanner.ci.vendors.CircleCi; +import org.sonar.scanner.ci.vendors.CirrusCi; +import org.sonar.scanner.ci.vendors.CodeMagic; +import org.sonar.scanner.ci.vendors.DroneCi; +import org.sonar.scanner.ci.vendors.GithubActions; +import org.sonar.scanner.ci.vendors.GitlabCi; +import org.sonar.scanner.ci.vendors.Jenkins; +import org.sonar.scanner.ci.vendors.SemaphoreCi; +import org.sonar.scanner.ci.vendors.TravisCi; +import org.sonar.scanner.cpd.CpdExecutor; +import org.sonar.scanner.cpd.CpdSettings; +import org.sonar.scanner.cpd.index.SonarCpdBlockIndex; +import org.sonar.scanner.issue.IssueFilters; +import org.sonar.scanner.issue.IssuePublisher; +import org.sonar.scanner.issue.ignore.EnforceIssuesFilter; +import org.sonar.scanner.issue.ignore.IgnoreIssuesFilter; +import org.sonar.scanner.issue.ignore.pattern.IssueExclusionPatternInitializer; +import org.sonar.scanner.issue.ignore.pattern.IssueInclusionPatternInitializer; +import org.sonar.scanner.issue.ignore.scanner.IssueExclusionsLoader; +import org.sonar.scanner.report.ActiveRulesPublisher; +import org.sonar.scanner.report.AnalysisCachePublisher; +import org.sonar.scanner.report.AnalysisContextReportPublisher; +import org.sonar.scanner.report.AnalysisWarningsPublisher; +import org.sonar.scanner.report.CeTaskReportDataHolder; +import org.sonar.scanner.report.ChangedLinesPublisher; +import org.sonar.scanner.report.ComponentsPublisher; +import org.sonar.scanner.report.ContextPropertiesPublisher; +import org.sonar.scanner.report.JavaArchitectureInformationProvider; +import org.sonar.scanner.report.MetadataPublisher; +import org.sonar.scanner.report.ReportPublisher; +import org.sonar.scanner.report.ScannerFileStructureProvider; +import org.sonar.scanner.report.SourcePublisher; +import org.sonar.scanner.report.TestExecutionPublisher; +import org.sonar.scanner.repository.ContextPropertiesCache; +import org.sonar.scanner.repository.DefaultProjectRepositoriesLoader; +import org.sonar.scanner.repository.DefaultQualityProfileLoader; +import org.sonar.scanner.repository.ProjectRepositoriesProvider; +import org.sonar.scanner.repository.QualityProfilesProvider; +import org.sonar.scanner.repository.ReferenceBranchSupplier; +import org.sonar.scanner.repository.language.DefaultLanguagesRepository; +import org.sonar.scanner.repository.language.LanguagesRepository; +import org.sonar.scanner.repository.settings.DefaultProjectSettingsLoader; +import org.sonar.scanner.rule.ActiveRulesProvider; +import org.sonar.scanner.rule.DefaultActiveRulesLoader; +import org.sonar.scanner.rule.QProfileVerifier; +import org.sonar.scanner.scan.DeprecatedPropertiesWarningGenerator; +import org.sonar.scanner.scan.InputModuleHierarchyProvider; +import org.sonar.scanner.scan.InputProjectProvider; +import org.sonar.scanner.scan.ModuleIndexer; +import org.sonar.scanner.scan.MutableProjectReactorProvider; +import org.sonar.scanner.scan.MutableProjectSettings; +import org.sonar.scanner.scan.ProjectBuildersExecutor; +import org.sonar.scanner.scan.ProjectConfigurationProvider; +import org.sonar.scanner.scan.ProjectLock; +import org.sonar.scanner.scan.ProjectReactorBuilder; +import org.sonar.scanner.scan.ProjectReactorValidator; +import org.sonar.scanner.scan.ProjectServerSettingsProvider; +import org.sonar.scanner.scan.ScanProperties; +import org.sonar.scanner.scan.SonarGlobalPropertiesFilter; +import org.sonar.scanner.scan.SpringProjectScanContainer; +import org.sonar.scanner.scan.WorkDirectoriesInitializer; +import org.sonar.scanner.scan.branch.BranchConfiguration; +import org.sonar.scanner.scan.branch.BranchConfigurationProvider; +import org.sonar.scanner.scan.branch.BranchType; +import org.sonar.scanner.scan.branch.ProjectBranchesProvider; +import org.sonar.scanner.scan.filesystem.DefaultProjectFileSystem; +import org.sonar.scanner.scan.filesystem.FileIndexer; +import org.sonar.scanner.scan.filesystem.InputComponentStore; +import org.sonar.scanner.scan.filesystem.LanguageDetection; +import org.sonar.scanner.scan.filesystem.MetadataGenerator; +import org.sonar.scanner.scan.filesystem.ProjectCoverageAndDuplicationExclusions; +import org.sonar.scanner.scan.filesystem.ProjectExclusionFilters; +import org.sonar.scanner.scan.filesystem.ProjectFileIndexer; +import org.sonar.scanner.scan.filesystem.ScannerComponentIdGenerator; +import org.sonar.scanner.scan.filesystem.StatusDetection; +import org.sonar.scanner.scan.measure.DefaultMetricFinder; +import org.sonar.scanner.scm.ScmChangedFilesProvider; +import org.sonar.scanner.scm.ScmConfiguration; +import org.sonar.scanner.scm.ScmPublisher; +import org.sonar.scanner.scm.ScmRevisionImpl; +import org.sonar.scanner.sensor.DefaultSensorStorage; +import org.sonar.scanner.sensor.ExecutingSensorContext; +import org.sonar.scanner.sensor.ProjectSensorContext; +import org.sonar.scanner.sensor.ProjectSensorOptimizer; +import org.sonar.scanner.sensor.UnchangedFilesHandler; +import org.sonar.scm.git.GitScmSupport; +import org.sonar.scm.svn.SvnScmSupport; + +import static org.sonar.api.batch.InstantiationStrategy.PER_BATCH; +import static org.sonar.core.extension.CoreExtensionsInstaller.noExtensionFilter; +import static org.sonar.scanner.bootstrap.ExtensionUtils.isDeprecatedScannerSide; +import static org.sonar.scanner.bootstrap.ExtensionUtils.isInstantiationStrategy; +import static org.sonar.scanner.bootstrap.ExtensionUtils.isScannerSide; + +@Priority(3) +public class SpringScannerContainer extends SpringComponentContainer { + private static final Logger LOG = LoggerFactory.getLogger(SpringScannerContainer.class); + + public SpringScannerContainer(SpringComponentContainer globalContainer) { + super(globalContainer); + } + + @Override + protected void doBeforeStart() { + addScannerExtensions(); + addComponents(); + } + + private void addScannerExtensions() { + getParentComponentByType(CoreExtensionsInstaller.class) + .install(this, noExtensionFilter(), extension -> getScannerProjectExtensionsFilter().accept(extension)); + getParentComponentByType(ExtensionInstaller.class) + .install(this, getScannerProjectExtensionsFilter()); + } + + private void addComponents() { + add( + ScanProperties.class, + ProjectReactorBuilder.class, + WorkDirectoriesInitializer.class, + new MutableProjectReactorProvider(), + ProjectBuildersExecutor.class, + ProjectLock.class, + ProjectReactorValidator.class, + ProjectInfo.class, + new BranchConfigurationProvider(), + new ProjectBranchesProvider(), + ProjectRepositoriesProvider.class, + new ProjectServerSettingsProvider(), + AnalysisCacheEnabled.class, + DeprecatedPropertiesWarningGenerator.class, + + // temp + new AnalysisTempFolderProvider(), + + // file system + ModuleIndexer.class, + InputComponentStore.class, + PathResolver.class, + new InputProjectProvider(), + new InputModuleHierarchyProvider(), + ScannerComponentIdGenerator.class, + new ScmChangedFilesProvider(), + StatusDetection.class, + LanguageDetection.class, + MetadataGenerator.class, + FileMetadata.class, + FileIndexer.class, + ProjectFileIndexer.class, + ProjectExclusionFilters.class, + + // rules + new ActiveRulesProvider(), + new QualityProfilesProvider(), + CheckFactory.class, + QProfileVerifier.class, + + // issues + DefaultNoSonarFilter.class, + IssueFilters.class, + IssuePublisher.class, + + // metrics + DefaultMetricFinder.class, + + // issue exclusions + IssueInclusionPatternInitializer.class, + IssueExclusionPatternInitializer.class, + IssueExclusionsLoader.class, + EnforceIssuesFilter.class, + IgnoreIssuesFilter.class, + + // context + ContextPropertiesCache.class, + + SensorStrategy.class, + + MutableProjectSettings.class, + SonarGlobalPropertiesFilter.class, + ProjectConfigurationProvider.class, + + ProjectCoverageAndDuplicationExclusions.class, + + // Plugin cache + AnalysisCacheProvider.class, + AnalysisCacheMemoryStorage.class, + DefaultAnalysisCacheLoader.class, + + // Report + ReferenceBranchSupplier.class, + ScannerMetrics.class, + JavaArchitectureInformationProvider.class, + ReportPublisher.class, + ScannerFileStructureProvider.class, + AnalysisContextReportPublisher.class, + MetadataPublisher.class, + ActiveRulesPublisher.class, + ComponentsPublisher.class, + ContextPropertiesPublisher.class, + AnalysisCachePublisher.class, + TestExecutionPublisher.class, + SourcePublisher.class, + ChangedLinesPublisher.class, + AnalysisWarningsPublisher.class, + + CeTaskReportDataHolder.class, + + // Cpd + CpdExecutor.class, + CpdSettings.class, + SonarCpdBlockIndex.class, + + // SCM + ScmConfiguration.class, + ScmPublisher.class, + ScmRevisionImpl.class, + + // Sensors + DefaultSensorStorage.class, + DefaultFileLinesContextFactory.class, + ProjectSensorContext.class, + ProjectSensorOptimizer.class, + ExecutingSensorContext.class, + + UnchangedFilesHandler.class, + + // Filesystem + DefaultProjectFileSystem.class, + + // CI + new CiConfigurationProvider(), + AppVeyor.class, + AwsCodeBuild.class, + AzureDevops.class, + Bamboo.class, + BitbucketPipelines.class, + Bitrise.class, + Buildkite.class, + CircleCi.class, + CirrusCi.class, + DroneCi.class, + GithubActions.class, + CodeMagic.class, + GitlabCi.class, + Jenkins.class, + SemaphoreCi.class, + TravisCi.class + ); + + add(GitScmSupport.getObjects()); + add(SvnScmSupport.getObjects()); + + add(DefaultProjectSettingsLoader.class, + DefaultActiveRulesLoader.class, + DefaultQualityProfileLoader.class, + DefaultProjectRepositoriesLoader.class); + + addIfMissing(DefaultLanguagesRepository.class, LanguagesRepository.class); + + } + + static ExtensionMatcher getScannerProjectExtensionsFilter() { + return extension -> { + if (isDeprecatedScannerSide(extension)) { + return isInstantiationStrategy(extension, PER_BATCH); + } + return isScannerSide(extension); + }; + } + + @Override + protected void doAfterStart() { + ScanProperties properties = getComponentByType(ScanProperties.class); + properties.validate(); + + properties.get("sonar.branch").ifPresent(deprecatedBranch -> { + throw MessageException.of("The 'sonar.branch' parameter is no longer supported. You should stop using it. " + + "Branch analysis is available in Developer Edition and above. See https://www.sonarsource.com/plans-and-pricing/developer/ for more information."); + }); + + BranchConfiguration branchConfig = getComponentByType(BranchConfiguration.class); + if (branchConfig.branchType() == BranchType.PULL_REQUEST) { + LOG.info("Pull request {} for merge into {} from {}", branchConfig.pullRequestKey(), pullRequestBaseToDisplayName(branchConfig.targetBranchName()), + branchConfig.branchName()); + } else if (branchConfig.branchName() != null) { + LOG.info("Branch name: {}", branchConfig.branchName()); + } + + getComponentByType(DeprecatedPropertiesWarningGenerator.class).execute(); + + getComponentByType(ProjectFileIndexer.class).index(); + new SpringProjectScanContainer(this).execute(); + } + + private static String pullRequestBaseToDisplayName(@Nullable String pullRequestBase) { + return pullRequestBase != null ? pullRequestBase : "default branch"; + } + +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakeLanguagesRepository.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakeLanguagesRepository.java new file mode 100644 index 00000000000..7443db29fa3 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakeLanguagesRepository.java @@ -0,0 +1,83 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.mediumtest; + +import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; +import javax.annotation.Priority; +import org.jetbrains.annotations.Nullable; +import org.sonar.api.resources.Languages; +import org.sonar.scanner.repository.language.Language; +import org.sonar.scanner.repository.language.LanguagesRepository; +import org.sonar.scanner.repository.language.SupportedLanguageDto; + +@Priority(1) +public class FakeLanguagesRepository implements LanguagesRepository { + + private final Map languageMap = new HashMap<>(); + + public FakeLanguagesRepository() { + languageMap.put("xoo", new Language(new FakeLanguage("xoo", "xoo", new String[] { ".xoo" }, new String[0], true))); + } + + public FakeLanguagesRepository(Languages languages) { + for (org.sonar.api.resources.Language language : languages.all()) { + languageMap.put(language.getKey(), new Language(new FakeLanguage(language.getKey(), language.getName(), language.getFileSuffixes(), language.filenamePatterns(), true))); + } + } + + @Nullable + @Override + public Language get(String languageKey) { + return languageMap.get(languageKey); + } + + @Override + public Collection all() { + return languageMap.values().stream() + // sorted for test consistency + .sorted(Comparator.comparing(Language::key)).toList(); + } + + public void addLanguage(String key, String name, String[] suffixes, String[] filenamePatterns) { + languageMap.put(key, new Language(new FakeLanguage(key, name, suffixes, filenamePatterns, true))); + } + + public void addLanguage(String key, String name, String[] suffixes, String[] filenamePatterns, boolean publishAllFiles) { + languageMap.put(key, new Language(new FakeLanguage(key, name, suffixes, filenamePatterns, publishAllFiles))); + } + + private static class FakeLanguage extends SupportedLanguageDto { + + private final boolean publishAllFiles; + + public FakeLanguage(String key, String name, String[] fileSuffixes, String[] filenamePatterns, boolean publishAllFiles) { + super(key, name, fileSuffixes, filenamePatterns); + this.publishAllFiles = publishAllFiles; + } + + @Override + public boolean publishAllFiles() { + return publishAllFiles; + } + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java index e5697c9b7b8..36e2bbbd298 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import javax.annotation.Priority; import org.sonar.api.Plugin; import org.sonar.core.platform.PluginInfo; @@ -48,7 +49,17 @@ public class FakePluginInstaller implements PluginInstaller { } @Override - public Map installRemotes() { + public Map installAllPlugins() { + return pluginsByKeys; + } + + @Override + public Map installRequiredPlugins() { + return pluginsByKeys; + } + + @Override + public Map installPluginsForLanguages(Set languageKeys) { return pluginsByKeys; } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/DefaultLanguagesRepository.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/DefaultLanguagesRepository.java index f5ab71c8e7f..607587950b6 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/DefaultLanguagesRepository.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/DefaultLanguagesRepository.java @@ -19,31 +19,74 @@ */ package org.sonar.scanner.repository.language; -import java.util.Arrays; +import com.google.gson.Gson; +import java.io.Reader; import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; import javax.annotation.CheckForNull; -import javax.annotation.concurrent.Immutable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.sonar.api.Startable; +import org.sonar.api.config.Configuration; import org.sonar.api.resources.Languages; +import org.sonar.scanner.bootstrap.DefaultScannerWsClient; +import org.sonarqube.ws.client.GetRequest; /** * Languages repository using {@link Languages} * @since 4.4 */ -@Immutable public class DefaultLanguagesRepository implements LanguagesRepository, Startable { + private static final Logger LOG = LoggerFactory.getLogger(DefaultLanguagesRepository.class); + private static final String LANGUAGES_WS_URL = "/api/languages/list"; + private static final Map PROPERTY_FRAGMENT_MAP = Map.of( + "js", "javascript", + "ts", "typescript", + "py", "python", + "web", "html" + ); - private final Languages languages; + private final Map languages = new HashMap<>(); + private final DefaultScannerWsClient wsClient; + private final Configuration properties; - public DefaultLanguagesRepository(Languages languages) { - this.languages = languages; + public DefaultLanguagesRepository(DefaultScannerWsClient wsClient, Configuration properties) { + this.wsClient = wsClient; + this.properties = properties; } @Override public void start() { - if (languages.all().length == 0) { - throw new IllegalStateException("No language plugins are installed."); + GetRequest getRequest = new GetRequest(LANGUAGES_WS_URL); + LanguagesWSResponse response; + try (Reader reader = wsClient.call(getRequest).contentReader()) { + response = new Gson().fromJson(reader, LanguagesWSResponse.class); + } catch (Exception e) { + throw new IllegalStateException("Fail to parse response of " + LANGUAGES_WS_URL, e); } + + languages.putAll(response.languages.stream() + .map(this::populateFileSuffixesAndPatterns) + .collect(Collectors.toMap(Language::key, Function.identity()))); + } + + private Language populateFileSuffixesAndPatterns(SupportedLanguageDto lang) { + String propertyFragment = PROPERTY_FRAGMENT_MAP.getOrDefault(lang.getKey(), lang.getKey()); + lang.setFileSuffixes(properties.getStringArray(String.format("sonar.%s.file.suffixes", propertyFragment))); + lang.setFilenamePatterns(properties.getStringArray(String.format("sonar.%s.file.patterns", propertyFragment))); + if (lang.filenamePatterns() == null && lang.getFileSuffixes() == null) { + LOG.debug("Language '{}' cannot be detected as it has neither suffixes nor patterns.", lang.getName()); + } + return new Language(lang); + } + + private String[] getFileSuffixes(String languageKey) { + String propName = String.format("sonar.%s.file.suffixes", PROPERTY_FRAGMENT_MAP.getOrDefault(languageKey, languageKey)); + return properties.getStringArray(propName); } /** @@ -52,8 +95,7 @@ public class DefaultLanguagesRepository implements LanguagesRepository, Startabl @Override @CheckForNull public Language get(String languageKey) { - org.sonar.api.resources.Language language = languages.get(languageKey); - return language != null ? new Language(language) : null; + return languages.get(languageKey); } /** @@ -61,9 +103,7 @@ public class DefaultLanguagesRepository implements LanguagesRepository, Startabl */ @Override public Collection all() { - return Arrays.stream(languages.all()) - .map(Language::new) - .toList(); + return languages.values(); } @Override @@ -71,4 +111,8 @@ public class DefaultLanguagesRepository implements LanguagesRepository, Startabl // nothing to do } + private static class LanguagesWSResponse { + List languages; + } + } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/SupportedLanguageDto.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/SupportedLanguageDto.java new file mode 100644 index 00000000000..bf9855e845c --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/SupportedLanguageDto.java @@ -0,0 +1,63 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.repository.language; + +public class SupportedLanguageDto implements org.sonar.api.resources.Language { + private String key; + private String name; + private String[] fileSuffixes; + private String[] filenamePatterns; + + public SupportedLanguageDto(String key, String name, String[] fileSuffixes, String[] filenamePatterns) { + this.key = key; + this.name = name; + this.fileSuffixes = fileSuffixes; + this.filenamePatterns = filenamePatterns; + } + + @Override + public String getKey() { + return key; + } + + @Override + public String getName() { + return name; + } + + @Override + public String[] getFileSuffixes() { + return fileSuffixes; + } + + public void setFileSuffixes(String[] fileSuffixes) { + this.fileSuffixes = fileSuffixes; + } + + @Override + public String[] filenamePatterns() { + return filenamePatterns; + } + + public void setFilenamePatterns(String[] filenamePatterns) { + this.filenamePatterns = filenamePatterns; + } + +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/settings/AbstractSettingsLoader.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/settings/AbstractSettingsLoader.java index e1bb0fa647e..6994ba5009e 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/settings/AbstractSettingsLoader.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/settings/AbstractSettingsLoader.java @@ -74,7 +74,9 @@ public abstract class AbstractSettingsLoader { static Map toMap(List settingsList) { Map result = new LinkedHashMap<>(); for (Settings.Setting s : settingsList) { - if (!s.getInherited()) { + // we need the "*.file.suffixes" and "*.file.patterns" properties for language detection + // see DefaultLanguagesRepository.populateFileSuffixesAndPatterns() + if (!s.getInherited() || s.getKey().endsWith(".file.suffixes") || s.getKey().endsWith(".file.patterns")) { switch (s.getValueOneOfCase()) { case VALUE: result.put(s.getKey(), s.getValue()); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java index cbd5f3faeaa..28a55fdaf98 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java @@ -19,124 +19,41 @@ */ package org.sonar.scanner.scan; -import javax.annotation.Nullable; +import java.util.Collection; +import java.util.Set; import javax.annotation.Priority; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.sonar.api.Plugin; import org.sonar.api.batch.fs.internal.DefaultInputModule; -import org.sonar.api.batch.fs.internal.FileMetadata; -import org.sonar.api.batch.fs.internal.SensorStrategy; -import org.sonar.api.batch.rule.CheckFactory; -import org.sonar.api.batch.sensor.issue.internal.DefaultNoSonarFilter; +import org.sonar.api.measures.Metrics; +import org.sonar.api.resources.Languages; import org.sonar.api.resources.ResourceTypes; -import org.sonar.api.scan.filesystem.PathResolver; -import org.sonar.api.utils.MessageException; import org.sonar.core.config.ScannerProperties; -import org.sonar.core.extension.CoreExtensionsInstaller; import org.sonar.core.language.LanguagesProvider; import org.sonar.core.metric.ScannerMetrics; +import org.sonar.core.platform.PluginInfo; import org.sonar.core.platform.SpringComponentContainer; -import org.sonar.scanner.DefaultFileLinesContextFactory; -import org.sonar.scanner.ProjectInfo; -import org.sonar.scanner.analysis.AnalysisTempFolderProvider; import org.sonar.scanner.bootstrap.ExtensionInstaller; import org.sonar.scanner.bootstrap.ExtensionMatcher; import org.sonar.scanner.bootstrap.GlobalAnalysisMode; import org.sonar.scanner.bootstrap.PostJobExtensionDictionary; -import org.sonar.scanner.cache.AnalysisCacheEnabled; -import org.sonar.scanner.cache.AnalysisCacheMemoryStorage; -import org.sonar.scanner.cache.AnalysisCacheProvider; -import org.sonar.scanner.cache.DefaultAnalysisCacheLoader; -import org.sonar.scanner.ci.CiConfigurationProvider; -import org.sonar.scanner.ci.vendors.AppVeyor; -import org.sonar.scanner.ci.vendors.AwsCodeBuild; -import org.sonar.scanner.ci.vendors.AzureDevops; -import org.sonar.scanner.ci.vendors.Bamboo; -import org.sonar.scanner.ci.vendors.BitbucketPipelines; -import org.sonar.scanner.ci.vendors.Bitrise; -import org.sonar.scanner.ci.vendors.Buildkite; -import org.sonar.scanner.ci.vendors.CircleCi; -import org.sonar.scanner.ci.vendors.CirrusCi; -import org.sonar.scanner.ci.vendors.CodeMagic; -import org.sonar.scanner.ci.vendors.DroneCi; -import org.sonar.scanner.ci.vendors.GithubActions; -import org.sonar.scanner.ci.vendors.GitlabCi; -import org.sonar.scanner.ci.vendors.Jenkins; -import org.sonar.scanner.ci.vendors.SemaphoreCi; -import org.sonar.scanner.ci.vendors.TravisCi; +import org.sonar.scanner.bootstrap.ScannerPluginRepository; import org.sonar.scanner.cpd.CpdExecutor; -import org.sonar.scanner.cpd.CpdSettings; -import org.sonar.scanner.cpd.index.SonarCpdBlockIndex; import org.sonar.scanner.fs.InputModuleHierarchy; -import org.sonar.scanner.issue.IssueFilters; -import org.sonar.scanner.issue.IssuePublisher; -import org.sonar.scanner.issue.ignore.EnforceIssuesFilter; -import org.sonar.scanner.issue.ignore.IgnoreIssuesFilter; -import org.sonar.scanner.issue.ignore.pattern.IssueExclusionPatternInitializer; -import org.sonar.scanner.issue.ignore.pattern.IssueInclusionPatternInitializer; -import org.sonar.scanner.issue.ignore.scanner.IssueExclusionsLoader; import org.sonar.scanner.mediumtest.AnalysisObservers; import org.sonar.scanner.postjob.DefaultPostJobContext; import org.sonar.scanner.postjob.PostJobOptimizer; import org.sonar.scanner.postjob.PostJobsExecutor; import org.sonar.scanner.qualitygate.QualityGateCheck; -import org.sonar.scanner.report.ActiveRulesPublisher; -import org.sonar.scanner.report.AnalysisCachePublisher; -import org.sonar.scanner.report.AnalysisContextReportPublisher; -import org.sonar.scanner.report.AnalysisWarningsPublisher; -import org.sonar.scanner.report.CeTaskReportDataHolder; -import org.sonar.scanner.report.ChangedLinesPublisher; -import org.sonar.scanner.report.ComponentsPublisher; -import org.sonar.scanner.report.ContextPropertiesPublisher; -import org.sonar.scanner.report.JavaArchitectureInformationProvider; -import org.sonar.scanner.report.MetadataPublisher; import org.sonar.scanner.report.ReportPublisher; -import org.sonar.scanner.report.ScannerFileStructureProvider; -import org.sonar.scanner.report.SourcePublisher; -import org.sonar.scanner.report.TestExecutionPublisher; -import org.sonar.scanner.repository.ContextPropertiesCache; -import org.sonar.scanner.repository.DefaultProjectRepositoriesLoader; -import org.sonar.scanner.repository.DefaultQualityProfileLoader; -import org.sonar.scanner.repository.ProjectRepositoriesProvider; -import org.sonar.scanner.repository.QualityProfilesProvider; -import org.sonar.scanner.repository.ReferenceBranchSupplier; -import org.sonar.scanner.repository.language.DefaultLanguagesRepository; -import org.sonar.scanner.repository.settings.DefaultProjectSettingsLoader; -import org.sonar.scanner.rule.ActiveRulesProvider; -import org.sonar.scanner.rule.DefaultActiveRulesLoader; import org.sonar.scanner.rule.QProfileVerifier; -import org.sonar.scanner.scan.branch.BranchConfiguration; -import org.sonar.scanner.scan.branch.BranchConfigurationProvider; -import org.sonar.scanner.scan.branch.BranchType; -import org.sonar.scanner.scan.branch.ProjectBranchesProvider; -import org.sonar.scanner.scan.filesystem.DefaultProjectFileSystem; -import org.sonar.scanner.scan.filesystem.FileIndexer; import org.sonar.scanner.scan.filesystem.InputComponentStore; -import org.sonar.scanner.scan.filesystem.LanguageDetection; -import org.sonar.scanner.scan.filesystem.MetadataGenerator; -import org.sonar.scanner.scan.filesystem.ProjectCoverageAndDuplicationExclusions; -import org.sonar.scanner.scan.filesystem.ProjectExclusionFilters; -import org.sonar.scanner.scan.filesystem.ProjectFileIndexer; -import org.sonar.scanner.scan.filesystem.ScannerComponentIdGenerator; -import org.sonar.scanner.scan.filesystem.StatusDetection; -import org.sonar.scanner.scan.measure.DefaultMetricFinder; -import org.sonar.scanner.scm.ScmChangedFilesProvider; -import org.sonar.scanner.scm.ScmConfiguration; import org.sonar.scanner.scm.ScmPublisher; -import org.sonar.scanner.scm.ScmRevisionImpl; -import org.sonar.scanner.sensor.DefaultSensorStorage; -import org.sonar.scanner.sensor.ExecutingSensorContext; -import org.sonar.scanner.sensor.ProjectSensorContext; import org.sonar.scanner.sensor.ProjectSensorExtensionDictionary; -import org.sonar.scanner.sensor.ProjectSensorOptimizer; import org.sonar.scanner.sensor.ProjectSensorsExecutor; -import org.sonar.scanner.sensor.UnchangedFilesHandler; -import org.sonar.scm.git.GitScmSupport; -import org.sonar.scm.svn.SvnScmSupport; import static org.sonar.api.batch.InstantiationStrategy.PER_BATCH; -import static org.sonar.api.utils.Preconditions.checkNotNull; -import static org.sonar.core.extension.CoreExtensionsInstaller.noExtensionFilter; import static org.sonar.scanner.bootstrap.ExtensionUtils.isDeprecatedScannerSide; import static org.sonar.scanner.bootstrap.ExtensionUtils.isInstantiationStrategy; import static org.sonar.scanner.bootstrap.ExtensionUtils.isScannerSide; @@ -145,182 +62,56 @@ import static org.sonar.scanner.bootstrap.ExtensionUtils.isScannerSide; public class SpringProjectScanContainer extends SpringComponentContainer { private static final Logger LOG = LoggerFactory.getLogger(SpringProjectScanContainer.class); - public SpringProjectScanContainer(SpringComponentContainer globalContainer) { - super(globalContainer); + public SpringProjectScanContainer(SpringComponentContainer parentContainer) { + super(parentContainer); } @Override protected void doBeforeStart() { - addScannerExtensions(); + Set languages = getParentComponentByType(InputComponentStore.class).languages(); + installPluginsForLanguages(languages); addScannerComponents(); } + private void installPluginsForLanguages(Set languageKeys) { + ScannerPluginRepository pluginRepository = getParentComponentByType(ScannerPluginRepository.class); + Collection languagePlugins = pluginRepository.installPluginsForLanguages(languageKeys); + for (PluginInfo pluginInfo : languagePlugins) { + Plugin instance = pluginRepository.getPluginInstance(pluginInfo.getKey()); + addExtension(pluginInfo, instance); + } + getParentComponentByType(ExtensionInstaller.class) + .installExtensionsForPlugins(this, getScannerProjectExtensionsFilter(), languagePlugins); + } + private void addScannerComponents() { add( - ScanProperties.class, - ProjectReactorBuilder.class, - WorkDirectoriesInitializer.class, - new MutableProjectReactorProvider(), - ProjectBuildersExecutor.class, - ProjectLock.class, - ResourceTypes.class, - ProjectReactorValidator.class, - ProjectInfo.class, - new BranchConfigurationProvider(), - new ProjectBranchesProvider(), - ProjectRepositoriesProvider.class, - new ProjectServerSettingsProvider(), - AnalysisCacheEnabled.class, - DeprecatedPropertiesWarningGenerator.class, - - // temp - new AnalysisTempFolderProvider(), - - // file system - ModuleIndexer.class, - InputComponentStore.class, - PathResolver.class, - new InputProjectProvider(), - new InputModuleHierarchyProvider(), - ScannerComponentIdGenerator.class, - new ScmChangedFilesProvider(), - StatusDetection.class, - LanguageDetection.class, - MetadataGenerator.class, - FileMetadata.class, - FileIndexer.class, - ProjectFileIndexer.class, - ProjectExclusionFilters.class, - - // rules - new ActiveRulesProvider(), - new QualityProfilesProvider(), - CheckFactory.class, - QProfileVerifier.class, - // issues - DefaultNoSonarFilter.class, - IssueFilters.class, - IssuePublisher.class, - - // metrics - DefaultMetricFinder.class, + ResourceTypes.class, // lang LanguagesProvider.class, - DefaultLanguagesRepository.class, - // issue exclusions - IssueInclusionPatternInitializer.class, - IssueExclusionPatternInitializer.class, - IssueExclusionsLoader.class, - EnforceIssuesFilter.class, - IgnoreIssuesFilter.class, - - // context - ContextPropertiesCache.class, - ContextPropertiesPublisher.class, + // rules + QProfileVerifier.class, - SensorStrategy.class, - MutableProjectSettings.class, ScannerProperties.class, - SonarGlobalPropertiesFilter.class, - ProjectConfigurationProvider.class, - - ProjectCoverageAndDuplicationExclusions.class, - - // Plugin cache - AnalysisCacheProvider.class, - AnalysisCacheMemoryStorage.class, - DefaultAnalysisCacheLoader.class, - - // Report - ReferenceBranchSupplier.class, - ScannerMetrics.class, - JavaArchitectureInformationProvider.class, - ReportPublisher.class, - ScannerFileStructureProvider.class, - AnalysisContextReportPublisher.class, - MetadataPublisher.class, - ActiveRulesPublisher.class, - ComponentsPublisher.class, - AnalysisCachePublisher.class, - TestExecutionPublisher.class, - SourcePublisher.class, - ChangedLinesPublisher.class, - AnalysisWarningsPublisher.class, - - CeTaskReportDataHolder.class, // QualityGate check QualityGateCheck.class, - // Cpd - CpdExecutor.class, - CpdSettings.class, - SonarCpdBlockIndex.class, - // PostJobs PostJobsExecutor.class, PostJobOptimizer.class, DefaultPostJobContext.class, PostJobExtensionDictionary.class, - // SCM - ScmConfiguration.class, - ScmPublisher.class, - ScmRevisionImpl.class, - // Sensors - DefaultSensorStorage.class, - DefaultFileLinesContextFactory.class, - ProjectSensorContext.class, - ProjectSensorOptimizer.class, - ProjectSensorsExecutor.class, - ExecutingSensorContext.class, ProjectSensorExtensionDictionary.class, - UnchangedFilesHandler.class, - - // Filesystem - DefaultProjectFileSystem.class, - - // CI - new CiConfigurationProvider(), - AppVeyor.class, - AwsCodeBuild.class, - AzureDevops.class, - Bamboo.class, - BitbucketPipelines.class, - Bitrise.class, - Buildkite.class, - CircleCi.class, - CirrusCi.class, - DroneCi.class, - GithubActions.class, - CodeMagic.class, - GitlabCi.class, - Jenkins.class, - SemaphoreCi.class, - TravisCi.class, + ProjectSensorsExecutor.class, AnalysisObservers.class); - - add(GitScmSupport.getObjects()); - add(SvnScmSupport.getObjects()); - - add(DefaultProjectSettingsLoader.class, - DefaultActiveRulesLoader.class, - DefaultQualityProfileLoader.class, - DefaultProjectRepositoriesLoader.class); - } - - private void addScannerExtensions() { - checkNotNull(getParent()); - getParent().getComponentByType(CoreExtensionsInstaller.class) - .install(this, noExtensionFilter(), extension -> getScannerProjectExtensionsFilter().accept(extension)); - getParent().getComponentByType(ExtensionInstaller.class) - .install(this, getScannerProjectExtensionsFilter()); } static ExtensionMatcher getScannerProjectExtensionsFilter() { @@ -334,29 +125,16 @@ public class SpringProjectScanContainer extends SpringComponentContainer { @Override protected void doAfterStart() { + getParentComponentByType(ScannerMetrics.class).addPluginMetrics(getComponentsByType(Metrics.class)); getComponentByType(ProjectLock.class).tryLock(); GlobalAnalysisMode analysisMode = getComponentByType(GlobalAnalysisMode.class); InputModuleHierarchy tree = getComponentByType(InputModuleHierarchy.class); ScanProperties properties = getComponentByType(ScanProperties.class); - properties.validate(); - - properties.get("sonar.branch").ifPresent(deprecatedBranch -> { - throw MessageException.of("The 'sonar.branch' parameter is no longer supported. You should stop using it. " + - "Branch analysis is available in Developer Edition and above. See https://www.sonarsource.com/plans-and-pricing/developer/ for more information."); - }); - BranchConfiguration branchConfig = getComponentByType(BranchConfiguration.class); - if (branchConfig.branchType() == BranchType.PULL_REQUEST) { - LOG.info("Pull request {} for merge into {} from {}", branchConfig.pullRequestKey(), pullRequestBaseToDisplayName(branchConfig.targetBranchName()), - branchConfig.branchName()); - } else if (branchConfig.branchName() != null) { - LOG.info("Branch name: {}", branchConfig.branchName()); + if (getComponentByType(Languages.class).all().length == 0) { + throw new IllegalStateException("No language plugins are installed."); } - getComponentByType(DeprecatedPropertiesWarningGenerator.class).execute(); - - getComponentByType(ProjectFileIndexer.class).index(); - // Log detected languages and their profiles after FS is indexed and languages detected getComponentByType(QProfileVerifier.class).execute(); @@ -382,10 +160,6 @@ public class SpringProjectScanContainer extends SpringComponentContainer { } } - private static String pullRequestBaseToDisplayName(@Nullable String pullRequestBase) { - return pullRequestBase != null ? pullRequestBase : "default branch"; - } - private void scanRecursively(InputModuleHierarchy tree, DefaultInputModule module) { for (DefaultInputModule child : tree.children(module)) { scanRecursively(tree, child); -- cgit v1.2.3