diff options
Diffstat (limited to 'sonar-scanner-engine/src/main/java/org/sonar')
43 files changed, 1443 insertions, 410 deletions
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/JGitCleanupService.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/JGitCleanupService.java new file mode 100644 index 00000000000..28e052cfa4e --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/JGitCleanupService.java @@ -0,0 +1,47 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.bootstrap; + +import java.lang.reflect.Method; +import org.eclipse.jgit.internal.util.CleanupService; + +/** + * Normally, JGit terminates with a shutdown hook. Since we also want to support running the Scanner Engine in the same JVM, this allows triggering shutdown manually. + */ +class JGitCleanupService implements AutoCloseable { + + private final Method shutDownMethod; + private final CleanupService cleanupService; + + public JGitCleanupService() { + cleanupService = new CleanupService(); + try { + shutDownMethod = CleanupService.class.getDeclaredMethod("shutDown"); + } catch (NoSuchMethodException e) { + throw new IllegalStateException("Unable to find method 'shutDown' on JGit CleanupService", e); + } + shutDownMethod.setAccessible(true); + } + + @Override + public void close() throws Exception { + shutDownMethod.invoke(cleanupService); + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerMain.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerMain.java index 0fe2c3ad479..bd8d5b9b99c 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerMain.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerMain.java @@ -21,17 +21,21 @@ package org.sonar.scanner.bootstrap; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.OutputStreamAppender; import com.google.gson.Gson; import com.google.gson.annotations.SerializedName; import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import javax.annotation.CheckForNull; +import javax.annotation.Nullable; import org.jetbrains.annotations.NotNull; import org.slf4j.LoggerFactory; import org.sonar.api.utils.MessageException; @@ -49,11 +53,13 @@ public class ScannerMain { private static final String SCANNER_APP_VERSION_KEY = "sonar.scanner.appVersion"; public static void main(String... args) { - System.exit(run(System.in)); + System.exit(run(System.in, System.out)); } - public static int run(InputStream in) { - try { + public static int run(InputStream in, OutputStream out) { + try (var ignored = new JGitCleanupService()) { + configureLogOutput(out); + LOG.info("Starting SonarScanner Engine..."); LOG.atInfo().log(ScannerMain::java); @@ -67,9 +73,11 @@ public class ScannerMain { LOG.info("SonarScanner Engine completed successfully"); return 0; - } catch (Exception e) { - handleException(e); + } catch (Throwable throwable) { + handleException(throwable); return 1; + } finally { + stopLogback(); } } @@ -87,30 +95,28 @@ public class ScannerMain { return sb.toString(); } - private static void handleException(Exception e) { - var messageException = unwrapMessageException(e); + private static void handleException(Throwable throwable) { + var messageException = unwrapMessageException(throwable); if (messageException.isPresent()) { // Don't show the stacktrace for a message exception to not pollute the logs if (LoggerFactory.getLogger(ScannerMain.class).isDebugEnabled()) { - LOG.error(messageException.get(), e); + LOG.error(messageException.get(), throwable); } else { LOG.error(messageException.get()); } } else { - LOG.error("Error during SonarScanner Engine execution", e); + LOG.error("Error during SonarScanner Engine execution", throwable); } } - private static Optional<String> unwrapMessageException(Exception t) { - Throwable y = t; - do { - if (y instanceof MessageException messageException) { - return Optional.of(messageException.getMessage()); - } - y = y.getCause(); - } while (y != null); - - return Optional.empty(); + private static Optional<String> unwrapMessageException(@Nullable Throwable throwable) { + if (throwable == null) { + return Optional.empty(); + } else if (throwable instanceof MessageException messageException) { + return Optional.of(messageException.getMessage()); + } else { + return unwrapMessageException(throwable.getCause()); + } } private static @NotNull Map<String, String> parseInputProperties(InputStream in) { @@ -157,6 +163,28 @@ public class ScannerMain { rootLogger.setLevel(Level.toLevel(verbose ? LEVEL_ROOT_VERBOSE : LEVEL_ROOT_DEFAULT)); } + private static void configureLogOutput(OutputStream out) { + var loggerContext = (ch.qos.logback.classic.LoggerContext) LoggerFactory.getILoggerFactory(); + var encoder = new ScannerLogbackEncoder(); + encoder.setContext(loggerContext); + encoder.start(); + + var appender = new OutputStreamAppender<ILoggingEvent>(); + appender.setEncoder(encoder); + appender.setContext(loggerContext); + appender.setOutputStream(out); + appender.start(); + + var rootLogger = (Logger) LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME); + rootLogger.addAppender(appender); + rootLogger.setLevel(Level.toLevel(LEVEL_ROOT_DEFAULT)); + } + + private static void stopLogback() { + var loggerContext = (ch.qos.logback.classic.LoggerContext) LoggerFactory.getILoggerFactory(); + loggerContext.stop(); + } + private static class Input { @SerializedName("scannerProperties") private List<ScannerProperty> scannerProperties; diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java index 3187c2e9aa3..194c1e17ffa 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java @@ -19,6 +19,10 @@ */ package org.sonar.scanner.bootstrap; +import static java.util.stream.Collectors.toMap; +import static org.sonar.api.utils.Preconditions.checkState; +import static org.sonar.core.config.ScannerProperties.PLUGIN_LOADING_OPTIMIZATION_KEY; + import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -38,10 +42,6 @@ import org.sonar.core.platform.PluginRepository; import org.sonar.core.plugin.PluginType; import org.sonar.scanner.mediumtest.LocalPlugin; -import static java.util.stream.Collectors.toMap; -import static org.sonar.api.utils.Preconditions.checkState; -import static org.sonar.core.config.ScannerProperties.PLUGIN_LOADING_OPTIMIZATION_KEY; - /** * Orchestrates the installation and loading of plugins */ @@ -83,7 +83,7 @@ public class ScannerPluginRepository implements PluginRepository, Startable { // this part is only used by medium tests for (LocalPlugin localPlugin : installer.installLocals()) { ScannerPlugin scannerPlugin = localPlugin.toScannerPlugin(); - String pluginKey = localPlugin.pluginKey(); + String pluginKey = localPlugin.pluginInfo().getKey(); pluginsByKeys.put(pluginKey, scannerPlugin); pluginInstancesByKeys.put(pluginKey, localPlugin.pluginInstance()); } @@ -112,7 +112,7 @@ public class ScannerPluginRepository implements PluginRepository, Startable { // this part is only used by medium tests for (LocalPlugin localPlugin : installer.installOptionalLocals(languageKeys)) { ScannerPlugin scannerPlugin = localPlugin.toScannerPlugin(); - String pluginKey = localPlugin.pluginKey(); + String pluginKey = localPlugin.pluginInfo().getKey(); languagePluginsByKeys.put(pluginKey, scannerPlugin); pluginsByKeys.put(pluginKey, scannerPlugin); pluginInstancesByKeys.put(pluginKey, localPlugin.pluginInstance()); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java index 29f1389e20c..133f4387856 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java @@ -26,7 +26,6 @@ import org.slf4j.LoggerFactory; import org.sonar.api.batch.fs.internal.FileMetadata; import org.sonar.api.batch.rule.CheckFactory; import org.sonar.api.batch.sensor.issue.internal.DefaultNoSonarFilter; -import org.sonar.api.config.PropertyDefinition; import org.sonar.api.scan.filesystem.PathResolver; import org.sonar.core.extension.CoreExtensionsInstaller; import org.sonar.core.metric.ScannerMetrics; @@ -87,6 +86,8 @@ import org.sonar.scanner.repository.ProjectRepositoriesProvider; import org.sonar.scanner.repository.QualityProfilesProvider; import org.sonar.scanner.repository.ReferenceBranchSupplier; import org.sonar.scanner.repository.TelemetryCache; +import org.sonar.scanner.repository.featureflags.DefaultFeatureFlagsLoader; +import org.sonar.scanner.repository.featureflags.DefaultFeatureFlagsRepository; import org.sonar.scanner.repository.language.DefaultLanguagesLoader; import org.sonar.scanner.repository.language.DefaultLanguagesRepository; import org.sonar.scanner.repository.settings.DefaultProjectSettingsLoader; @@ -98,7 +99,6 @@ import org.sonar.scanner.scan.InputModuleHierarchyProvider; import org.sonar.scanner.scan.InputProjectProvider; import org.sonar.scanner.scan.ModuleIndexer; import org.sonar.scanner.scan.MutableProjectReactorProvider; -import org.sonar.scanner.scan.MutableProjectSettings; import org.sonar.scanner.scan.ProjectBuildersExecutor; import org.sonar.scanner.scan.ProjectConfigurationProvider; import org.sonar.scanner.scan.ProjectLock; @@ -115,6 +115,7 @@ import org.sonar.scanner.scan.branch.BranchType; import org.sonar.scanner.scan.branch.ProjectBranchesProvider; import org.sonar.scanner.scan.filesystem.DefaultProjectFileSystem; import org.sonar.scanner.scan.filesystem.FilePreprocessor; +import org.sonar.scanner.scan.filesystem.HiddenFilesProjectData; import org.sonar.scanner.scan.filesystem.InputComponentStore; import org.sonar.scanner.scan.filesystem.LanguageDetection; import org.sonar.scanner.scan.filesystem.MetadataGenerator; @@ -152,25 +153,10 @@ public class SpringScannerContainer extends SpringComponentContainer { @Override protected void doBeforeStart() { - addSuffixesDeprecatedProperties(); addScannerExtensions(); addComponents(); } - private void addSuffixesDeprecatedProperties() { - add( - /* - * This is needed to support properly the deprecated sonar.rpg.suffixes property when the download optimization feature is enabled. - * The value of the property is needed at the preprocessing stage, but being defined by an optional analyzer means that at preprocessing - * it won't be properly available. This will be removed in SQ 11.0 together with the drop of the property from the rpg analyzer. - * See SONAR-21514 - */ - PropertyDefinition.builder("sonar.rpg.file.suffixes") - .deprecatedKey("sonar.rpg.suffixes") - .multiValues(true) - .build()); - } - private void addScannerExtensions() { getParentComponentByType(CoreExtensionsInstaller.class) .install(this, noExtensionFilter(), extension -> getScannerProjectExtensionsFilter().accept(extension)); @@ -214,6 +200,7 @@ public class SpringScannerContainer extends SpringComponentContainer { FilePreprocessor.class, ProjectFilePreprocessor.class, ProjectExclusionFilters.class, + HiddenFilesProjectData.class, // rules new ActiveRulesProvider(), @@ -240,7 +227,6 @@ public class SpringScannerContainer extends SpringComponentContainer { ContextPropertiesCache.class, TelemetryCache.class, - MutableProjectSettings.class, SonarGlobalPropertiesFilter.class, ProjectConfigurationProvider.class, @@ -309,17 +295,20 @@ public class SpringScannerContainer extends SpringComponentContainer { GitlabCi.class, Jenkins.class, SemaphoreCi.class, - TravisCi.class); - - add(GitScmSupport.getObjects()); - add(SvnScmSupport.getObjects()); + TravisCi.class, - add(DefaultProjectSettingsLoader.class, + DefaultProjectSettingsLoader.class, DefaultActiveRulesLoader.class, DefaultQualityProfileLoader.class, DefaultProjectRepositoriesLoader.class, DefaultLanguagesLoader.class, - DefaultLanguagesRepository.class); + DefaultLanguagesRepository.class, + + DefaultFeatureFlagsLoader.class, + DefaultFeatureFlagsRepository.class); + + add(GitScmSupport.getObjects()); + add(SvnScmSupport.getObjects()); } static ExtensionMatcher getScannerProjectExtensionsFilter() { diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RulesSeverityDetector.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RulesSeverityDetector.java index 703cd038fd0..d4f22ad9704 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RulesSeverityDetector.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RulesSeverityDetector.java @@ -97,8 +97,11 @@ public class RulesSeverityDetector { } private static Map<String, Result.Level> getDriverDefinedRuleSeverities(Run run) { - return run.getTool().getDriver().getRules() - .stream() + Set<ReportingDescriptor> rules = run.getTool().getDriver().getRules(); + if (rules == null) { + return emptyMap(); + } + return rules.stream() .filter(RulesSeverityDetector::hasRuleDefinedLevel) .collect(toMap(ReportingDescriptor::getId, x -> Result.Level.valueOf(x.getDefaultConfiguration().getLevel().name()))); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RunMapper.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RunMapper.java index 5b9abf383cf..bdf5a9a1114 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RunMapper.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RunMapper.java @@ -83,7 +83,7 @@ public class RunMapper { private List<NewAdHocRule> toNewAdHocRules(Run run, String driverName, Map<String, Result.Level> ruleSeveritiesByRuleId, Map<String, Result.Level> ruleSeveritiesByRuleIdForNewCCT) { - Set<ReportingDescriptor> driverRules = run.getTool().getDriver().getRules(); + Set<ReportingDescriptor> driverRules = Optional.ofNullable(run.getTool().getDriver().getRules()).orElse(Set.of()); Set<ReportingDescriptor> extensionRules = hasExtensions(run.getTool()) ? run.getTool().getExtensions().stream().filter(RunMapper::hasRules).flatMap(extension -> extension.getRules().stream()).collect(toSet()) : Set.of(); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/http/ScannerWsClientProvider.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/http/ScannerWsClientProvider.java index 088ebfb0052..d9eed8bedc8 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/http/ScannerWsClientProvider.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/http/ScannerWsClientProvider.java @@ -87,7 +87,7 @@ public class ScannerWsClientProvider { String responseTimeout = defaultIfBlank(scannerProps.property(SONAR_SCANNER_RESPONSE_TIMEOUT), valueOf(DEFAULT_RESPONSE_TIMEOUT)); String envVarToken = defaultIfBlank(system.envVariable(TOKEN_ENV_VARIABLE), null); String token = defaultIfBlank(scannerProps.property(TOKEN_PROPERTY), envVarToken); - String login = defaultIfBlank(scannerProps.property(CoreProperties.LOGIN), token); + String login = defaultIfBlank(token, scannerProps.property(CoreProperties.LOGIN)); boolean skipSystemTrustMaterial = Boolean.parseBoolean(defaultIfBlank(scannerProps.property(SKIP_SYSTEM_TRUST_MATERIAL), "false")); var sslContext = configureSsl(parseSslConfig(scannerProps, sonarUserHome), system, skipSystemTrustMaterial); connectorBuilder diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/issue/IssuePublisher.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/issue/IssuePublisher.java index 1e37a715066..df9bfd00b48 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/issue/IssuePublisher.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/issue/IssuePublisher.java @@ -24,6 +24,7 @@ import java.util.Collection; import java.util.EnumMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Consumer; import javax.annotation.Nullable; import javax.annotation.concurrent.ThreadSafe; @@ -54,6 +55,7 @@ import org.sonar.scanner.report.ReportPublisher; @ThreadSafe public class IssuePublisher { + private static final Set<String> noSonarKeyContains = Set.of("nosonar", "S1291"); private final ActiveRules activeRules; private final IssueFilters filters; private final ReportPublisher reportPublisher; @@ -91,7 +93,7 @@ public class IssuePublisher { return inputComponent.isFile() && textRange != null && ((DefaultInputFile) inputComponent).hasNoSonarAt(textRange.start().line()) - && !StringUtils.containsIgnoreCase(issue.ruleKey().rule(), "nosonar"); + && noSonarKeyContains.stream().noneMatch(k -> StringUtils.containsIgnoreCase(issue.ruleKey().rule(), k)); } public void initAndAddExternalIssue(ExternalIssue issue) { diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/LocalPlugin.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/LocalPlugin.java index 0a3d42979f7..70df99a4f3d 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/LocalPlugin.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/LocalPlugin.java @@ -25,9 +25,13 @@ import org.sonar.core.platform.PluginInfo; import org.sonar.core.plugin.PluginType; import org.sonar.scanner.bootstrap.ScannerPlugin; -public record LocalPlugin(String pluginKey, Plugin pluginInstance, Set<String> requiredForLanguages) { +public record LocalPlugin(PluginInfo pluginInfo, Plugin pluginInstance, Set<String> requiredForLanguages) { + + public LocalPlugin(String pluginKey, Plugin pluginInstance, Set<String> requiredForLanguages) { + this(new PluginInfo(pluginKey).setOrganizationName("SonarSource"), pluginInstance, requiredForLanguages); + } public ScannerPlugin toScannerPlugin() { - return new ScannerPlugin(pluginKey, 1L, PluginType.BUNDLED, new PluginInfo(pluginKey)); + return new ScannerPlugin(pluginInfo.getKey(), 1L, PluginType.BUNDLED, pluginInfo); } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/DefaultFeatureFlagsLoader.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/DefaultFeatureFlagsLoader.java new file mode 100644 index 00000000000..8eb338f3fba --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/DefaultFeatureFlagsLoader.java @@ -0,0 +1,54 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.repository.featureflags; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import java.io.Reader; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import org.sonar.scanner.http.ScannerWsClient; +import org.sonarqube.ws.client.GetRequest; + +public class DefaultFeatureFlagsLoader implements FeatureFlagsLoader { + + private static final String FEATURE_FLAGS_WS_URL = "/api/features/list"; + + private final ScannerWsClient wsClient; + + public DefaultFeatureFlagsLoader(ScannerWsClient wsClient) { + this.wsClient = wsClient; + } + + @Override + public Set<String> load() { + GetRequest getRequest = new GetRequest(FEATURE_FLAGS_WS_URL); + List<String> jsonResponse; + try (Reader reader = wsClient.call(getRequest).contentReader()) { + jsonResponse = new Gson().fromJson(reader, new TypeToken<ArrayList<String>>() { + }.getType()); + } catch (Exception e) { + throw new IllegalStateException("Unable to load feature flags", e); + } + return Set.copyOf(jsonResponse); + } + +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/DefaultFeatureFlagsRepository.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/DefaultFeatureFlagsRepository.java new file mode 100644 index 00000000000..08b52011ed3 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/DefaultFeatureFlagsRepository.java @@ -0,0 +1,49 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.repository.featureflags; + +import java.util.HashSet; +import java.util.Set; +import org.sonar.api.Startable; + +public class DefaultFeatureFlagsRepository implements FeatureFlagsRepository, Startable { + + private final Set<String> featureFlags = new HashSet<>(); + private final FeatureFlagsLoader featureFlagsLoader; + + public DefaultFeatureFlagsRepository(FeatureFlagsLoader featureFlagsLoader) { + this.featureFlagsLoader = featureFlagsLoader; + } + + @Override + public void start() { + featureFlags.addAll(featureFlagsLoader.load()); + } + + @Override + public void stop() { + // nothing to do + } + + @Override + public boolean isEnabled(String flagName) { + return featureFlags.contains(flagName); + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/FeatureFlag.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/FeatureFlag.java new file mode 100644 index 00000000000..034b2a7c098 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/FeatureFlag.java @@ -0,0 +1,23 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.repository.featureflags; + +public record FeatureFlag(String flagName, boolean enabled) { +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/FeatureFlagsLoader.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/FeatureFlagsLoader.java new file mode 100644 index 00000000000..71213e86394 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/FeatureFlagsLoader.java @@ -0,0 +1,28 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.repository.featureflags; + +import java.util.Set; + +public interface FeatureFlagsLoader { + + Set<String> load(); + +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/FeatureFlagsRepository.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/FeatureFlagsRepository.java new file mode 100644 index 00000000000..f354cd1c0f0 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/FeatureFlagsRepository.java @@ -0,0 +1,26 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.repository.featureflags; + +public interface FeatureFlagsRepository { + + boolean isEnabled(String flagName); + +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/package-info.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/package-info.java new file mode 100644 index 00000000000..4b27773b5c7 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/package-info.java @@ -0,0 +1,23 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.scanner.repository.featureflags; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliCacheService.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliCacheService.java new file mode 100644 index 00000000000..24db0ddec64 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliCacheService.java @@ -0,0 +1,250 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.sca; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.io.UncheckedIOException; +import java.lang.reflect.Type; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.internal.apachecommons.lang3.SystemUtils; +import org.sonar.api.utils.System2; +import org.sonar.scanner.bootstrap.SonarUserHome; +import org.sonar.scanner.http.ScannerWsClient; +import org.sonar.scanner.repository.TelemetryCache; +import org.sonarqube.ws.client.GetRequest; +import org.sonarqube.ws.client.WsResponse; + +import static java.lang.String.format; + +/** + * This class is responsible for checking the SQ server for the latest version of the CLI, + * caching the CLI for use across different projects, updating the cached CLI to the latest + * version, and holding on to the cached CLI's file location so that other service classes + * can make use of it. + */ +public class CliCacheService { + protected static final String CLI_WS_URL = "api/v2/sca/clis"; + private static final Logger LOG = LoggerFactory.getLogger(CliCacheService.class); + private final SonarUserHome sonarUserHome; + private final ScannerWsClient wsClient; + private final TelemetryCache telemetryCache; + private final System2 system2; + + public CliCacheService(SonarUserHome sonarUserHome, ScannerWsClient wsClient, TelemetryCache telemetryCache, System2 system2) { + this.sonarUserHome = sonarUserHome; + this.wsClient = wsClient; + this.telemetryCache = telemetryCache; + this.system2 = system2; + } + + static Path newTempFile(Path tempDir) { + try { + return Files.createTempFile(tempDir, "scaFileCache", null); + } catch (IOException e) { + throw new IllegalStateException("Fail to create temp file in " + tempDir, e); + } + } + + static void moveFile(Path sourceFile, Path targetFile) { + try { + Files.move(sourceFile, targetFile, StandardCopyOption.ATOMIC_MOVE); + } catch (IOException e1) { + // Check if the file was cached by another process during download + if (!Files.exists(targetFile)) { + LOG.warn("Unable to rename {} to {}", sourceFile, targetFile); + LOG.warn("A copy/delete will be tempted but with no guarantee of atomicity"); + try { + Files.move(sourceFile, targetFile); + } catch (IOException e2) { + throw new IllegalStateException("Fail to move " + sourceFile + " to " + targetFile, e2); + } + } + } + } + + static void mkdir(Path dir) { + try { + Files.createDirectories(dir); + } catch (IOException e) { + throw new IllegalStateException("Fail to create cache directory: " + dir, e); + } + } + + static void downloadBinaryTo(Path downloadLocation, WsResponse response) { + try (InputStream stream = response.contentStream()) { + FileUtils.copyInputStreamToFile(stream, downloadLocation.toFile()); + } catch (IOException e) { + throw new IllegalStateException(format("Fail to download SCA CLI into %s", downloadLocation), e); + } + } + + public File cacheCli() { + boolean success = false; + + var alternateLocation = system2.envVariable("TIDELIFT_CLI_LOCATION"); + if (alternateLocation != null) { + LOG.info("Using alternate location for Tidelift CLI: {}", alternateLocation); + // If the TIDELIFT_CLI_LOCATION environment variable is set, we should use that location + // instead of trying to download the CLI from the server. + File cliFile = new File(alternateLocation); + if (!cliFile.exists()) { + throw new IllegalStateException(format("Alternate location for Tidelift CLI has been set but no file was found at %s", alternateLocation)); + } + return cliFile; + } + + try { + List<CliMetadataResponse> metadataResponses = getLatestMetadata(apiOsName(), apiArch()); + + if (metadataResponses.isEmpty()) { + throw new IllegalStateException(format("Could not find CLI for %s %s", apiOsName(), apiArch())); + } + + // We should only be getting one matching CLI for the OS + Arch combination. + // If we have more than one CLI to choose from then I'm not sure which one to choose. + if (metadataResponses.size() > 1) { + throw new IllegalStateException("Multiple CLI matches found. Unable to correctly cache CLI."); + } + + CliMetadataResponse metadataResponse = metadataResponses.get(0); + String checksum = metadataResponse.sha256(); + // If we have a matching checksum dir with the existing CLI file, then we are up to date. + if (!cachedCliFile(checksum).exists()) { + LOG.debug("SCA CLI update detected"); + downloadCli(metadataResponse.id(), checksum); + telemetryCache.put("scanner.sca.get.cli.cache.hit", "false"); + } else { + telemetryCache.put("scanner.sca.get.cli.cache.hit", "true"); + } + + File cliFile = cachedCliFile(checksum); + success = true; + return cliFile; + } finally { + telemetryCache.put("scanner.sca.get.cli.success", String.valueOf(success)); + } + } + + Path cacheDir() { + return sonarUserHome.getPath().resolve("cache"); + } + + private File cachedCliFile(String checksum) { + return cacheDir().resolve(checksum).resolve(fileName()).toFile(); + } + + private String fileName() { + return system2.isOsWindows() ? "tidelift.exe" : "tidelift"; + } + + private List<CliMetadataResponse> getLatestMetadata(String osName, String arch) { + LOG.info("Requesting CLI for OS {} and arch {}", osName, arch); + GetRequest getRequest = new GetRequest(CLI_WS_URL).setParam("os", osName).setParam("arch", arch); + try (WsResponse response = wsClient.call(getRequest)) { + try (Reader reader = response.contentReader()) { + Type listOfMetadata = new TypeToken<ArrayList<CliMetadataResponse>>() { + }.getType(); + return new Gson().fromJson(reader, listOfMetadata); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private void downloadCli(String id, String checksum) { + LOG.info("Downloading cli {}", id); + long startTime = system2.now(); + boolean success = false; + GetRequest getRequest = new GetRequest(CLI_WS_URL + "/" + id).setHeader("Accept", "application/octet-stream"); + + try (WsResponse response = wsClient.call(getRequest)) { + // Download to a temporary file location in case another process is also trying to + // create the CLI file in the checksum cache directory. Once the file is downloaded to a temporary + // location, do an atomic move to the correct cache location. + Path tempDir = createTempDir(); + Path tempFile = newTempFile(tempDir); + downloadBinaryTo(tempFile, response); + File destinationFile = cachedCliFile(checksum); + // We need to make sure the folder structure exists for the correct cache location before performing the move. + mkdir(destinationFile.toPath().getParent()); + moveFile(tempFile, destinationFile.toPath()); + if (!destinationFile.setExecutable(true, false)) { + throw new IllegalStateException("Unable to mark CLI as executable"); + } + success = true; + } catch (Exception e) { + throw new IllegalStateException("Unable to download CLI executable", e); + } finally { + telemetryCache.put("scanner.sca.download.cli.duration", String.valueOf(system2.now() - startTime)); + telemetryCache.put("scanner.sca.download.cli.success", String.valueOf(success)); + } + } + + String apiOsName() { + // We don't want to send the raw OS name because there could be too many combinations of the OS name + // to reliably match up with the correct CLI needed to be downloaded. Instead, we send a subset of + // OS names that should match to the correct CLI here. + if (system2.isOsWindows()) { + return "windows"; + } else if (system2.isOsMac()) { + return "mac"; + } else { + return "linux"; + } + } + + String apiArch() { + return SystemUtils.OS_ARCH.toLowerCase(Locale.ENGLISH); + } + + Path createTempDir() { + Path dir = sonarUserHome.getPath().resolve("_tmp"); + try { + if (Files.exists(dir)) { + return dir; + } else { + return Files.createDirectory(dir); + } + } catch (IOException e) { + throw new IllegalStateException("Unable to create temp directory at " + dir, e); + } + } + + private record CliMetadataResponse( + String id, + String filename, + String sha256, + String os, + String arch) { + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliService.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliService.java new file mode 100644 index 00000000000..6b3418a8f6a --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliService.java @@ -0,0 +1,214 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.sca; + +import java.io.File; +import java.io.IOException; +import java.io.StringWriter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.stream.Stream; +import javax.annotation.Nullable; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.event.Level; +import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.api.platform.Server; +import org.sonar.api.utils.System2; +import org.sonar.core.util.ProcessWrapperFactory; +import org.sonar.scanner.config.DefaultConfiguration; +import org.sonar.scanner.repository.TelemetryCache; +import org.sonar.scanner.scan.filesystem.ProjectExclusionFilters; +import org.sonar.scanner.scm.ScmConfiguration; +import org.sonar.scm.git.JGitUtils; + +/** + * The CliService class is meant to serve as the main entrypoint for any commands + * that should be executed by the CLI. It will handle manages the external process, + * raising any errors that happen while running a command, and passing back the + * data generated by the command to the caller. + */ +public class CliService { + private static final Logger LOG = LoggerFactory.getLogger(CliService.class); + public static final String SCA_EXCLUSIONS_KEY = "sonar.sca.exclusions"; + public static final String LEGACY_SCA_EXCLUSIONS_KEY = "sonar.sca.excludedManifests"; + + private final ProcessWrapperFactory processWrapperFactory; + private final TelemetryCache telemetryCache; + private final System2 system2; + private final Server server; + private final ScmConfiguration scmConfiguration; + private final ProjectExclusionFilters projectExclusionFilters; + + public CliService(ProcessWrapperFactory processWrapperFactory, TelemetryCache telemetryCache, System2 system2, Server server, ScmConfiguration scmConfiguration, + ProjectExclusionFilters projectExclusionFilters) { + this.processWrapperFactory = processWrapperFactory; + this.telemetryCache = telemetryCache; + this.system2 = system2; + this.server = server; + this.scmConfiguration = scmConfiguration; + this.projectExclusionFilters = projectExclusionFilters; + } + + public File generateManifestsArchive(DefaultInputModule module, File cliExecutable, DefaultConfiguration configuration) throws IOException, IllegalStateException { + long startTime = system2.now(); + boolean success = false; + try { + String archiveName = "dependency-files.tar.xz"; + Path archivePath = module.getWorkDir().resolve(archiveName); + List<String> args = new ArrayList<>(); + args.add(cliExecutable.getAbsolutePath()); + args.add("projects"); + args.add("save-lockfiles"); + args.add("--xz"); + args.add("--xz-filename"); + args.add(archivePath.toAbsolutePath().toString()); + args.add("--directory"); + args.add(module.getBaseDir().toString()); + args.add("--recursive"); + + String excludeFlag = getExcludeFlag(module, configuration); + if (excludeFlag != null) { + args.add("--exclude"); + args.add(excludeFlag); + } + + if (LOG.isDebugEnabled()) { + LOG.info("Setting CLI to debug mode"); + args.add("--debug"); + } + + Map<String, String> envProperties = new HashMap<>(); + // sending this will tell the CLI to skip checking for the latest available version on startup + envProperties.put("TIDELIFT_SKIP_UPDATE_CHECK", "1"); + envProperties.put("TIDELIFT_ALLOW_MANIFEST_FAILURES", "1"); + envProperties.put("TIDELIFT_CLI_INSIDE_SCANNER_ENGINE", "1"); + envProperties.put("TIDELIFT_CLI_SQ_SERVER_VERSION", server.getVersion()); + envProperties.putAll(ScaProperties.buildFromScannerProperties(configuration)); + + LOG.info("Running command: {}", args); + LOG.info("Environment properties: {}", envProperties); + + Consumer<String> logConsumer = LOG.atLevel(Level.INFO)::log; + processWrapperFactory.create(module.getWorkDir(), logConsumer, logConsumer, envProperties, args.toArray(new String[0])).execute(); + LOG.info("Generated manifests archive file: {}", archiveName); + success = true; + return archivePath.toFile(); + } finally { + telemetryCache.put("scanner.sca.execution.cli.duration", String.valueOf(system2.now() - startTime)); + telemetryCache.put("scanner.sca.execution.cli.success", String.valueOf(success)); + } + } + + private @Nullable String getExcludeFlag(DefaultInputModule module, DefaultConfiguration configuration) throws IOException { + List<String> configExcludedPaths = getConfigExcludedPaths(configuration, projectExclusionFilters); + List<String> scmIgnoredPaths = getScmIgnoredPaths(module); + + ArrayList<String> mergedExclusionPaths = new ArrayList<>(); + mergedExclusionPaths.addAll(configExcludedPaths); + mergedExclusionPaths.addAll(scmIgnoredPaths); + + String workDirExcludedPath = getWorkDirExcludedPath(module); + if (workDirExcludedPath != null) { + mergedExclusionPaths.add(workDirExcludedPath); + } + + if (mergedExclusionPaths.isEmpty()) { + return null; + } + + // wrap each exclusion path in quotes to handle commas in file paths + return toCsvString(mergedExclusionPaths); + } + + private static List<String> getConfigExcludedPaths(DefaultConfiguration configuration, ProjectExclusionFilters projectExclusionFilters) { + String[] sonarExclusions = projectExclusionFilters.getExclusionsConfig(InputFile.Type.MAIN); + String[] scaExclusions = configuration.getStringArray(SCA_EXCLUSIONS_KEY); + String[] scaExclusionsLegacy = configuration.getStringArray(LEGACY_SCA_EXCLUSIONS_KEY); + + return Stream.of(sonarExclusions, scaExclusions, scaExclusionsLegacy) + .flatMap(Arrays::stream) + .distinct() + .toList(); + } + + private List<String> getScmIgnoredPaths(DefaultInputModule module) { + var scmProvider = scmConfiguration.provider(); + // Only Git is supported at this time + if (scmProvider == null || scmProvider.key() == null || !scmProvider.key().equals("git")) { + return List.of(); + } + + if (scmConfiguration.isExclusionDisabled()) { + // The user has opted out of using the SCM exclusion rules + return List.of(); + } + + Path baseDirPath = module.getBaseDir(); + List<String> scmIgnoredPaths = JGitUtils.getAllIgnoredPaths(baseDirPath); + if (scmIgnoredPaths.isEmpty()) { + return List.of(); + } + return scmIgnoredPaths.stream() + .map(ignoredPathRel -> { + + boolean isDirectory = false; + try { + isDirectory = Files.isDirectory(baseDirPath.resolve(ignoredPathRel.replace("/", File.separator))); + } catch (java.nio.file.InvalidPathException e) { + // if it's not a valid path, it's not a directory so we can just pass to the Tidelift CLI + } + // Directories need to get turned into a glob for the Tidelift CLI + return isDirectory ? (ignoredPathRel + "/**") : ignoredPathRel; + }) + .toList(); + } + + private static String getWorkDirExcludedPath(DefaultInputModule module) { + Path baseDir = module.getBaseDir().toAbsolutePath().normalize(); + Path workDir = module.getWorkDir().toAbsolutePath().normalize(); + + if (workDir.startsWith(baseDir)) { + // workDir is inside baseDir, so return the relative path as a glob + Path relativeWorkDir = baseDir.relativize(workDir); + return relativeWorkDir + "/**"; + } + + return null; + } + + private static String toCsvString(List<String> values) throws IOException { + StringWriter sw = new StringWriter(); + try (CSVPrinter printer = new CSVPrinter(sw, CSVFormat.DEFAULT)) { + printer.printRecord(values); + } + // trim to remove the trailing newline + return sw.toString().trim(); + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaExecutor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaExecutor.java new file mode 100644 index 00000000000..143e144c2dc --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaExecutor.java @@ -0,0 +1,92 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.sca; + +import java.io.File; +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.time.StopWatch; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.scanner.config.DefaultConfiguration; +import org.sonar.scanner.report.ReportPublisher; +import org.sonar.scanner.repository.featureflags.FeatureFlagsRepository; + +/** + * The ScaExecutor class is the main entrypoint for generating manifest dependency + * data during a Sonar scan and passing that data in the report so that it can + * be analyzed further by SQ server. + */ +public class ScaExecutor { + private static final Logger LOG = LoggerFactory.getLogger(ScaExecutor.class); + private static final String SCA_FEATURE_NAME = "sca"; + + private final CliCacheService cliCacheService; + private final CliService cliService; + private final ReportPublisher reportPublisher; + private final FeatureFlagsRepository featureFlagsRepository; + private final DefaultConfiguration configuration; + + public ScaExecutor(CliCacheService cliCacheService, CliService cliService, ReportPublisher reportPublisher, FeatureFlagsRepository featureFlagsRepository, + DefaultConfiguration configuration) { + this.cliCacheService = cliCacheService; + this.cliService = cliService; + this.reportPublisher = reportPublisher; + this.featureFlagsRepository = featureFlagsRepository; + this.configuration = configuration; + } + + public void execute(DefaultInputModule root) { + // Global feature flag + if (!featureFlagsRepository.isEnabled(SCA_FEATURE_NAME)) { + LOG.info("Dependency analysis skipped"); + return; + } + + // Project or scanner level feature flag + if (!configuration.getBoolean("sonar.sca.enabled").orElse(true)) { + LOG.info("Dependency analysis disabled for this project"); + return; + } + + var stopwatch = new StopWatch(); + stopwatch.start(); + LOG.info("Checking for latest CLI"); + File cliFile = cliCacheService.cacheCli(); + + LOG.info("Collecting manifests for the dependency analysis..."); + if (cliFile.exists()) { + try { + File generatedZip = cliService.generateManifestsArchive(root, cliFile, configuration); + LOG.debug("Zip ready for report: {}", generatedZip); + reportPublisher.getWriter().writeScaFile(generatedZip); + LOG.debug("Manifest zip written to report"); + } catch (IOException | IllegalStateException e) { + LOG.error("Error gathering manifests", e); + } finally { + stopwatch.stop(); + if (LOG.isInfoEnabled()) { + LOG.info("Load SCA project dependencies (done) | time={}ms", stopwatch.getTime(TimeUnit.MILLISECONDS)); + } + } + } + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaProperties.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaProperties.java new file mode 100644 index 00000000000..a697aef3e20 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaProperties.java @@ -0,0 +1,82 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.sca; + +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import org.sonar.scanner.config.DefaultConfiguration; + +public class ScaProperties { + private static final Pattern sonarScaPropertyRegex = Pattern.compile("^sonar\\.sca\\.([a-zA-Z]+)$"); + private static final String SONAR_SCA_PREFIX = "sonar.sca."; + private static final Set<String> IGNORED_PROPERTIES = Set.of( + // sonar.sca.exclusions is a special case which we handle when building --exclude + "sonar.sca.exclusions", + // excludedManifests is a special case which we handle when building --exclude + "sonar.sca.excludedManifests", + // keep recursive enabled to better match sonar-scanner behavior + "sonar.sca.recursiveManifestSearch"); + + private ScaProperties() { + } + + /** + * Build a map of environment variables from the sonar.sca.* properties in the configuration. + * The environment variable names are derived from the property names by removing the sonar.sca. prefix + * and converting to upper snake case to be used with the Tidelift CLI with the value from the configuration. + * <p> + * Examples: + * <br> + * { "sonar.sca.propertyName" : "value" } becomes { "TIDELIFT_PROPERTY_NAME" : "value" } + * <br> + * { "sonar.someOtherProperty" : "value" } returns an empty map + * + * @param configuration the scanner configuration possibly containing sonar.sca.* properties + * @return a map of Tidelift CLI compatible environment variable names to their configuration values + */ + public static Map<String, String> buildFromScannerProperties(DefaultConfiguration configuration) { + HashMap<String, String> props = new HashMap<>(configuration.getProperties()); + + return props + .entrySet() + .stream() + .filter(entry -> entry.getKey().startsWith(SONAR_SCA_PREFIX)) + .filter(entry -> !IGNORED_PROPERTIES.contains(entry.getKey())) + .collect(Collectors.toMap(entry -> convertPropToEnvVariable(entry.getKey()), Map.Entry::getValue)); + } + + // convert sonar.sca.* to TIDELIFT_* and convert from camelCase to UPPER_SNAKE_CASE + private static String convertPropToEnvVariable(String propertyName) { + var regexMatcher = sonarScaPropertyRegex.matcher(propertyName); + + if (regexMatcher.matches() && regexMatcher.groupCount() == 1) { + var tideliftNamespace = "TIDELIFT_"; + var convertedPropertyName = PropertyNamingStrategies.UpperSnakeCaseStrategy.INSTANCE.translate(regexMatcher.group(1)); + + return tideliftNamespace + convertedPropertyName; + } + + return propertyName; + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/package-info.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/package-info.java new file mode 100644 index 00000000000..b0f34909c27 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/package-info.java @@ -0,0 +1,23 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.scanner.sca; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/MutableModuleSettings.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/MutableModuleSettings.java deleted file mode 100644 index 15912f8a510..00000000000 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/MutableModuleSettings.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2025 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.scanner.scan; - -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import jakarta.annotation.Priority; -import org.sonar.api.config.internal.Settings; - -import static java.util.Objects.requireNonNull; - -/** - * @deprecated since 6.5 {@link ModuleConfiguration} used to be mutable, so keep a mutable copy for backward compatibility. - */ -@Deprecated -@Priority(1) -public class MutableModuleSettings extends Settings { - - private final Map<String, String> properties = new HashMap<>(); - - public MutableModuleSettings(ModuleConfiguration config) { - super(config.getDefinitions(), config.getEncryption()); - addProperties(config.getProperties()); - } - - @Override - protected Optional<String> get(String key) { - return Optional.ofNullable(properties.get(key)); - } - - @Override - protected void set(String key, String value) { - properties.put( - requireNonNull(key, "key can't be null"), - requireNonNull(value, "value can't be null").trim()); - } - - @Override - protected void remove(String key) { - properties.remove(key); - } - - @Override - public Map<String, String> getProperties() { - return properties; - } -} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/MutableProjectSettings.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/MutableProjectSettings.java deleted file mode 100644 index df24cbe81e5..00000000000 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/MutableProjectSettings.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2025 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.scanner.scan; - -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import org.sonar.api.config.internal.Settings; -import org.sonar.scanner.bootstrap.GlobalConfiguration; - -import jakarta.annotation.Priority; - -import static java.util.Objects.requireNonNull; - -/** - * @deprecated since 6.5 {@link ProjectConfiguration} used to be mutable, so keep a mutable copy for backward compatibility. - */ -@Deprecated -@Priority(2) -public class MutableProjectSettings extends Settings { - - private final Map<String, String> properties = new HashMap<>(); - - public MutableProjectSettings(GlobalConfiguration globalConfig) { - super(globalConfig.getDefinitions(), globalConfig.getEncryption()); - addProperties(globalConfig.getProperties()); - } - - public void complete(ProjectConfiguration projectConfig) { - addProperties(projectConfig.getProperties()); - } - - @Override - protected Optional<String> get(String key) { - return Optional.ofNullable(properties.get(key)); - } - - @Override - protected void set(String key, String value) { - properties.put( - requireNonNull(key, "key can't be null"), - requireNonNull(value, "value can't be null").trim()); - } - - @Override - protected void remove(String key) { - properties.remove(key); - } - - @Override - public Map<String, String> getProperties() { - return properties; - } -} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectConfigurationProvider.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectConfigurationProvider.java index c12ec245924..e5543d4f9c5 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectConfigurationProvider.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectConfigurationProvider.java @@ -26,7 +26,6 @@ import org.sonar.scanner.bootstrap.GlobalConfiguration; import org.sonar.scanner.bootstrap.GlobalServerSettings; import org.springframework.context.annotation.Bean; - public class ProjectConfigurationProvider { private final SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter; @@ -37,7 +36,7 @@ public class ProjectConfigurationProvider { @Bean("ProjectConfiguration") public ProjectConfiguration provide(DefaultInputProject project, GlobalConfiguration globalConfig, GlobalServerSettings globalServerSettings, - ProjectServerSettings projectServerSettings, MutableProjectSettings projectSettings) { + ProjectServerSettings projectServerSettings) { Map<String, String> settings = new LinkedHashMap<>(); settings.putAll(globalServerSettings.properties()); settings.putAll(projectServerSettings.properties()); @@ -45,10 +44,7 @@ public class ProjectConfigurationProvider { settings = sonarGlobalPropertiesFilter.enforceOnlyServerSideSonarGlobalPropertiesAreUsed(settings, globalServerSettings.properties()); - ProjectConfiguration projectConfig = new ProjectConfiguration(globalConfig.getDefinitions(), globalConfig.getEncryption(), settings); - projectSettings.complete(projectConfig); - return projectConfig; + return new ProjectConfiguration(globalConfig.getDefinitions(), globalConfig.getEncryption(), settings); } - } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringModuleScanContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringModuleScanContainer.java index 4315c762481..8ddb889912d 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringModuleScanContainer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringModuleScanContainer.java @@ -54,7 +54,6 @@ public class SpringModuleScanContainer extends SpringComponentContainer { add( module.definition(), module, - MutableModuleSettings.class, SonarGlobalPropertiesFilter.class, ModuleConfigurationProvider.class, @@ -68,8 +67,7 @@ public class SpringModuleScanContainer extends SpringComponentContainer { ModuleSensorOptimizer.class, ModuleSensorContext.class, - ModuleSensorExtensionDictionary.class - ); + ModuleSensorExtensionDictionary.class); } private void addExtensions() { diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java index ef57ea5a076..ead791bdeaf 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java @@ -19,9 +19,9 @@ */ package org.sonar.scanner.scan; +import jakarta.annotation.Priority; import java.util.Collection; import java.util.Set; -import jakarta.annotation.Priority; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.api.Plugin; @@ -52,6 +52,9 @@ import org.sonar.scanner.postjob.PostJobsExecutor; import org.sonar.scanner.qualitygate.QualityGateCheck; import org.sonar.scanner.report.ReportPublisher; import org.sonar.scanner.rule.QProfileVerifier; +import org.sonar.scanner.sca.CliCacheService; +import org.sonar.scanner.sca.CliService; +import org.sonar.scanner.sca.ScaExecutor; import org.sonar.scanner.scan.filesystem.FileIndexer; import org.sonar.scanner.scan.filesystem.InputFileFilterRepository; import org.sonar.scanner.scan.filesystem.LanguageDetection; @@ -131,7 +134,12 @@ public class SpringProjectScanContainer extends SpringComponentContainer { // file system InputFileFilterRepository.class, FileIndexer.class, - ProjectFileIndexer.class); + ProjectFileIndexer.class, + + // SCA + CliService.class, + CliCacheService.class, + ScaExecutor.class); } static ExtensionMatcher getScannerProjectExtensionsFilter() { @@ -172,6 +180,9 @@ public class SpringProjectScanContainer extends SpringComponentContainer { LOG.info("------------- Run sensors on project"); getComponentByType(ProjectSensorsExecutor.class).execute(); + LOG.info("------------- Gather SCA dependencies on project"); + getComponentByType(ScaExecutor.class).execute(tree.root()); + getComponentByType(ScmPublisher.class).publish(); getComponentByType(CpdExecutor.class).execute(); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java index 314e923ce71..242bc015574 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java @@ -24,17 +24,15 @@ import java.nio.file.AccessDeniedException; import java.nio.file.FileSystemLoopException; import java.nio.file.FileVisitResult; import java.nio.file.FileVisitor; -import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.attribute.DosFileAttributes; -import org.apache.commons.lang3.SystemUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DefaultInputModule; import org.sonar.scanner.fs.InputModuleHierarchy; +import org.sonar.scanner.scan.ModuleConfiguration; public class DirectoryFileVisitor implements FileVisitor<Path> { @@ -43,27 +41,31 @@ public class DirectoryFileVisitor implements FileVisitor<Path> { private final FileVisitAction fileVisitAction; private final DefaultInputModule module; private final ModuleExclusionFilters moduleExclusionFilters; - private final InputModuleHierarchy inputModuleHierarchy; private final InputFile.Type type; + private final HiddenFilesVisitorHelper hiddenFilesVisitorHelper; - DirectoryFileVisitor(FileVisitAction fileVisitAction, DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, - InputModuleHierarchy inputModuleHierarchy, InputFile.Type type) { + DirectoryFileVisitor(FileVisitAction fileVisitAction, DefaultInputModule module, ModuleConfiguration moduleConfig, ModuleExclusionFilters moduleExclusionFilters, + InputModuleHierarchy inputModuleHierarchy, InputFile.Type type, HiddenFilesProjectData hiddenFilesProjectData) { this.fileVisitAction = fileVisitAction; this.module = module; this.moduleExclusionFilters = moduleExclusionFilters; this.inputModuleHierarchy = inputModuleHierarchy; this.type = type; + this.hiddenFilesVisitorHelper = new HiddenFilesVisitorHelper(hiddenFilesProjectData, module, moduleConfig); } @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - return isHidden(dir) ? FileVisitResult.SKIP_SUBTREE : FileVisitResult.CONTINUE; + if (hiddenFilesVisitorHelper.shouldVisitDir(dir)) { + return FileVisitResult.CONTINUE; + } + return FileVisitResult.SKIP_SUBTREE; } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - if (!Files.isHidden(file)) { + if (hiddenFilesVisitorHelper.shouldVisitFile(file)) { fileVisitAction.execute(file); } return FileVisitResult.CONTINUE; @@ -129,25 +131,12 @@ public class DirectoryFileVisitor implements FileVisitor<Path> { @Override public FileVisitResult postVisitDirectory(Path dir, IOException exc) { + hiddenFilesVisitorHelper.exitDirectory(dir); return FileVisitResult.CONTINUE; } - private static boolean isHidden(Path path) throws IOException { - if (SystemUtils.IS_OS_WINDOWS) { - try { - DosFileAttributes dosFileAttributes = Files.readAttributes(path, DosFileAttributes.class, LinkOption.NOFOLLOW_LINKS); - return dosFileAttributes.isHidden(); - } catch (UnsupportedOperationException e) { - return path.toFile().isHidden(); - } - } else { - return Files.isHidden(path); - } - } - @FunctionalInterface interface FileVisitAction { void execute(Path file) throws IOException; } } - diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java index 7f31c949132..0961edbd985 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java @@ -63,12 +63,13 @@ public class FileIndexer { private final ModuleRelativePathWarner moduleRelativePathWarner; private final InputFileFilterRepository inputFileFilterRepository; private final Languages languages; + private final HiddenFilesProjectData hiddenFilesProjectData; public FileIndexer(DefaultInputProject project, ScannerComponentIdGenerator scannerComponentIdGenerator, InputComponentStore componentStore, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, IssueExclusionsLoader issueExclusionsLoader, MetadataGenerator metadataGenerator, SensorStrategy sensorStrategy, LanguageDetection languageDetection, ScanProperties properties, ScmChangedFiles scmChangedFiles, StatusDetection statusDetection, ModuleRelativePathWarner moduleRelativePathWarner, - InputFileFilterRepository inputFileFilterRepository, Languages languages) { + InputFileFilterRepository inputFileFilterRepository, Languages languages, HiddenFilesProjectData hiddenFilesProjectData) { this.project = project; this.scannerComponentIdGenerator = scannerComponentIdGenerator; this.componentStore = componentStore; @@ -83,15 +84,18 @@ public class FileIndexer { this.moduleRelativePathWarner = moduleRelativePathWarner; this.inputFileFilterRepository = inputFileFilterRepository; this.languages = languages; + this.hiddenFilesProjectData = hiddenFilesProjectData; } - void indexFile(DefaultInputModule module, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path sourceFile, - Type type, ProgressReport progressReport) { + void indexFile(DefaultInputModule module, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path sourceFile, Type type, + ProgressReport progressReport) { Path projectRelativePath = project.getBaseDir().relativize(sourceFile); Path moduleRelativePath = module.getBaseDir().relativize(sourceFile); // This should be fast; language should be cached from preprocessing step Language language = langDetection.language(sourceFile, projectRelativePath); + // cached from directory file visitation, after querying the data is removed to reduce memory consumption + boolean isHidden = hiddenFilesProjectData.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(sourceFile, module); DefaultIndexedFile indexedFile = new DefaultIndexedFile( sourceFile, @@ -102,11 +106,12 @@ public class FileIndexer { language != null ? language.key() : null, scannerComponentIdGenerator.getAsInt(), sensorStrategy, - scmChangedFiles.getOldRelativeFilePath(sourceFile)); + scmChangedFiles.getOldRelativeFilePath(sourceFile), + isHidden); DefaultInputFile inputFile = new DefaultInputFile(indexedFile, f -> metadataGenerator.setMetadata(module.key(), f, module.getEncoding()), f -> f.setStatus(statusDetection.findStatusFromScm(f))); - if (language != null && isPublishAllFiles(language.key())) { + if (!isHidden && language != null && isPublishAllFiles(language.key())) { inputFile.setPublished(true); } if (!accept(inputFile)) { diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java index 544fe46c43b..a87c5f11fc9 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java @@ -147,22 +147,35 @@ public class FilePreprocessor { return true; } - Path target = Files.readSymbolicLink(absolutePath); - if (!Files.exists(target)) { + Optional<Path> target = resolvePathToTarget(absolutePath); + if (target.isEmpty() || !Files.exists(target.get())) { LOG.warn("File '{}' is ignored. It is a symbolic link targeting a file that does not exist.", absolutePath); return false; } - if (!target.startsWith(project.getBaseDir())) { + if (!target.get().startsWith(project.getBaseDir())) { LOG.warn("File '{}' is ignored. It is a symbolic link targeting a file not located in project basedir.", absolutePath); return false; } - if (!target.startsWith(moduleBaseDirectory)) { + if (!target.get().startsWith(moduleBaseDirectory)) { LOG.info("File '{}' is ignored. It is a symbolic link targeting a file not located in module basedir.", absolutePath); return false; } return true; } + + private static Optional<Path> resolvePathToTarget(Path symbolicLinkAbsolutePath) throws IOException { + Path target = Files.readSymbolicLink(symbolicLinkAbsolutePath); + if (target.isAbsolute()) { + return Optional.of(target); + } + + try { + return Optional.of(symbolicLinkAbsolutePath.getParent().resolve(target).toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize()); + } catch (IOException e) { + return Optional.empty(); + } + } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectData.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectData.java new file mode 100644 index 00000000000..d779a054455 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectData.java @@ -0,0 +1,77 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.scan.filesystem; + +import java.io.IOException; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import org.apache.commons.lang3.SystemUtils; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.scanner.bootstrap.SonarUserHome; + +public class HiddenFilesProjectData { + + final Map<DefaultInputModule, Set<Path>> hiddenFilesByModule = new HashMap<>(); + private final SonarUserHome sonarUserHome; + private Path cachedSonarUserHomePath; + + public HiddenFilesProjectData(SonarUserHome sonarUserHome) { + this.sonarUserHome = sonarUserHome; + } + + public void markAsHiddenFile(Path file, DefaultInputModule module) { + hiddenFilesByModule.computeIfAbsent(module, k -> new HashSet<>()).add(file); + } + + /** + * To alleviate additional strain on the memory, we remove the visibility information for <code>hiddenFilesByModule</code> mapdirectly after querying, + * as we don't need it afterward. + */ + public boolean getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(Path file, DefaultInputModule module) { + Set<Path> hiddenFilesPerModule = hiddenFilesByModule.get(module); + if (hiddenFilesPerModule != null) { + return hiddenFilesPerModule.remove(file); + } + return false; + } + + public Path getCachedSonarUserHomePath() throws IOException { + if (cachedSonarUserHomePath == null) { + cachedSonarUserHomePath = resolveRealPath(sonarUserHome.getPath()); + } + return cachedSonarUserHomePath; + } + + public void clearHiddenFilesData() { + // Allowing the GC to collect the map, should only be done after all indexing is complete + hiddenFilesByModule.clear(); + } + + public Path resolveRealPath(Path path) throws IOException { + if (SystemUtils.IS_OS_WINDOWS) { + return path.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize(); + } + return path; + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesVisitorHelper.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesVisitorHelper.java new file mode 100644 index 00000000000..607a859ef44 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesVisitorHelper.java @@ -0,0 +1,112 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.scan.filesystem; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.attribute.DosFileAttributes; +import org.apache.commons.lang3.SystemUtils; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.scanner.scan.ModuleConfiguration; + +public class HiddenFilesVisitorHelper { + + private static final String EXCLUDE_HIDDEN_FILES_PROPERTY = "sonar.scanner.excludeHiddenFiles"; + private final HiddenFilesProjectData hiddenFilesProjectData; + private final DefaultInputModule module; + final boolean excludeHiddenFiles; + private Path moduleWorkDir; + Path rootHiddenDir; + + public HiddenFilesVisitorHelper(HiddenFilesProjectData hiddenFilesProjectData, DefaultInputModule module, ModuleConfiguration moduleConfig) { + this.hiddenFilesProjectData = hiddenFilesProjectData; + this.module = module; + this.excludeHiddenFiles = moduleConfig.getBoolean(EXCLUDE_HIDDEN_FILES_PROPERTY).orElse(false); + } + + public boolean shouldVisitDir(Path path) throws IOException { + boolean isHidden = isHiddenDir(path); + + if (isHidden && (excludeHiddenFiles || isExcludedHiddenDirectory(path))) { + return false; + } + if (isHidden) { + enterHiddenDirectory(path); + } + return true; + } + + private boolean isExcludedHiddenDirectory(Path path) throws IOException { + return getCachedModuleWorkDir().equals(path) || hiddenFilesProjectData.getCachedSonarUserHomePath().equals(path); + } + + void enterHiddenDirectory(Path dir) { + if (!insideHiddenDirectory()) { + rootHiddenDir = dir; + } + } + + public void exitDirectory(Path path) { + if (insideHiddenDirectory() && rootHiddenDir.equals(path)) { + resetRootHiddenDir(); + } + } + + void resetRootHiddenDir() { + this.rootHiddenDir = null; + } + + public boolean shouldVisitFile(Path path) throws IOException { + boolean isHidden = insideHiddenDirectory() || Files.isHidden(path); + + if (!excludeHiddenFiles && isHidden) { + hiddenFilesProjectData.markAsHiddenFile(path, module); + } + + return !excludeHiddenFiles || !isHidden; + } + + private Path getCachedModuleWorkDir() throws IOException { + if (moduleWorkDir == null) { + moduleWorkDir = hiddenFilesProjectData.resolveRealPath(module.getWorkDir()); + } + return moduleWorkDir; + } + + // visible for testing + boolean insideHiddenDirectory() { + return rootHiddenDir != null; + } + + protected static boolean isHiddenDir(Path path) throws IOException { + if (SystemUtils.IS_OS_WINDOWS) { + try { + DosFileAttributes dosFileAttributes = Files.readAttributes(path, DosFileAttributes.class, LinkOption.NOFOLLOW_LINKS); + return dosFileAttributes.isHidden(); + } catch (UnsupportedOperationException e) { + return path.toFile().isHidden(); + } + } else { + return Files.isHidden(path); + } + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStore.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStore.java index 6ef26dafd07..68b6d1db580 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStore.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStore.java @@ -19,12 +19,15 @@ */ package org.sonar.scanner.scan.filesystem; +import java.util.Set; import java.util.SortedSet; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import org.sonar.api.batch.ScannerSide; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.InputModule; -import org.sonar.api.batch.fs.internal.SensorStrategy; import org.sonar.api.batch.fs.internal.DefaultFileSystem; +import org.sonar.api.batch.fs.internal.SensorStrategy; @ScannerSide public class ModuleInputComponentStore extends DefaultFileSystem.Cache { @@ -73,11 +76,29 @@ public class ModuleInputComponentStore extends DefaultFileSystem.Cache { @Override public Iterable<InputFile> getFilesByName(String filename) { - return inputComponentStore.getFilesByName(filename); + Iterable<InputFile> allFilesByName = inputComponentStore.getFilesByName(filename); + if (strategy.isGlobal()) { + return allFilesByName; + } + + return filterByModule(allFilesByName); } @Override public Iterable<InputFile> getFilesByExtension(String extension) { - return inputComponentStore.getFilesByExtension(extension); + Iterable<InputFile> allFilesByExtension = inputComponentStore.getFilesByExtension(extension); + if (strategy.isGlobal()) { + return allFilesByExtension; + } + + return filterByModule(allFilesByExtension); + } + + private Iterable<InputFile> filterByModule(Iterable<InputFile> projectInputFiles) { + Set<InputFile> projectInputFilesSet = StreamSupport.stream(projectInputFiles.spliterator(), false) + .collect(Collectors.toSet()); + return StreamSupport.stream(inputComponentStore.filesByModule(moduleKey).spliterator(), false) + .filter(projectInputFilesSet::contains) + .toList(); } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/MutableFileSystem.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/MutableFileSystem.java index 5daa384d3ac..9c969f6ae20 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/MutableFileSystem.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/MutableFileSystem.java @@ -25,35 +25,54 @@ import org.sonar.api.batch.fs.FilePredicates; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DefaultFileSystem; import org.sonar.api.batch.fs.internal.predicates.ChangedFilePredicate; +import org.sonar.api.batch.fs.internal.predicates.NonHiddenFilesPredicate; public class MutableFileSystem extends DefaultFileSystem { - private boolean restrictToChangedFiles = false; + + boolean restrictToChangedFiles = false; + boolean allowHiddenFileAnalysis = false; public MutableFileSystem(Path baseDir, Cache cache, FilePredicates filePredicates) { super(baseDir, cache, filePredicates); } - public MutableFileSystem(Path baseDir) { + MutableFileSystem(Path baseDir) { super(baseDir); } @Override public Iterable<InputFile> inputFiles(FilePredicate requestPredicate) { - if (restrictToChangedFiles) { - return super.inputFiles(new ChangedFilePredicate(requestPredicate)); - } - return super.inputFiles(requestPredicate); + return super.inputFiles(applyAdditionalPredicate(requestPredicate)); } @Override public InputFile inputFile(FilePredicate requestPredicate) { + return super.inputFile(applyAdditionalPredicate(requestPredicate)); + } + + private FilePredicate applyAdditionalPredicate(FilePredicate requestPredicate) { + return applyHiddenFilePredicate(applyChangedFilePredicate(requestPredicate)); + } + + private FilePredicate applyHiddenFilePredicate(FilePredicate predicate) { + if (allowHiddenFileAnalysis) { + return predicate; + } + return predicates().and(new NonHiddenFilesPredicate(), predicate); + } + + private FilePredicate applyChangedFilePredicate(FilePredicate predicate) { if (restrictToChangedFiles) { - return super.inputFile(new ChangedFilePredicate(requestPredicate)); + return predicates().and(new ChangedFilePredicate(), predicate); } - return super.inputFile(requestPredicate); + return predicate; } public void setRestrictToChangedFiles(boolean restrictToChangedFiles) { this.restrictToChangedFiles = restrictToChangedFiles; } + + public void setAllowHiddenFileAnalysis(boolean allowHiddenFileAnalysis) { + this.allowHiddenFileAnalysis = allowHiddenFileAnalysis; + } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java index 97e449fcb26..c1349872c24 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java @@ -62,6 +62,7 @@ public class ProjectFileIndexer { private final FileIndexer fileIndexer; private final ProjectFilePreprocessor projectFilePreprocessor; private final AnalysisWarnings analysisWarnings; + private final HiddenFilesProjectData hiddenFilesProjectData; private ProgressReport progressReport; @@ -69,7 +70,7 @@ public class ProjectFileIndexer { SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter, InputModuleHierarchy inputModuleHierarchy, GlobalConfiguration globalConfig, GlobalServerSettings globalServerSettings, ProjectServerSettings projectServerSettings, FileIndexer fileIndexer, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, - ProjectFilePreprocessor projectFilePreprocessor, AnalysisWarnings analysisWarnings) { + ProjectFilePreprocessor projectFilePreprocessor, AnalysisWarnings analysisWarnings, HiddenFilesProjectData hiddenFilesProjectData) { this.componentStore = componentStore; this.sonarGlobalPropertiesFilter = sonarGlobalPropertiesFilter; this.inputModuleHierarchy = inputModuleHierarchy; @@ -81,6 +82,7 @@ public class ProjectFileIndexer { this.projectCoverageAndDuplicationExclusions = projectCoverageAndDuplicationExclusions; this.projectFilePreprocessor = projectFilePreprocessor; this.analysisWarnings = analysisWarnings; + this.hiddenFilesProjectData = hiddenFilesProjectData; } public void index() { @@ -91,10 +93,10 @@ public class ProjectFileIndexer { projectCoverageAndDuplicationExclusions.log(" "); indexModulesRecursively(inputModuleHierarchy.root()); + hiddenFilesProjectData.clearHiddenFilesData(); int totalIndexed = componentStore.inputFiles().size(); - progressReport.stop(totalIndexed + " " + pluralizeFiles(totalIndexed) + " indexed"); - + progressReport.stopAndLogTotalTime(totalIndexed + " " + pluralizeFiles(totalIndexed) + " indexed"); } private void indexModulesRecursively(DefaultInputModule module) { @@ -118,15 +120,15 @@ public class ProjectFileIndexer { moduleCoverageAndDuplicationExclusions.log(" "); } List<Path> mainSourceDirsOrFiles = projectFilePreprocessor.getMainSourcesByModule(module); - indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, mainSourceDirsOrFiles, Type.MAIN); + indexFiles(module, moduleConfig, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, mainSourceDirsOrFiles, Type.MAIN); projectFilePreprocessor.getTestSourcesByModule(module) - .ifPresent(tests -> indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, tests, Type.TEST)); + .ifPresent(tests -> indexFiles(module, moduleConfig, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, tests, Type.TEST)); } private static void logPaths(String label, Path baseDir, List<Path> paths) { if (!paths.isEmpty()) { StringBuilder sb = new StringBuilder(label); - for (Iterator<Path> it = paths.iterator(); it.hasNext(); ) { + for (Iterator<Path> it = paths.iterator(); it.hasNext();) { Path file = it.next(); Optional<String> relativePathToBaseDir = PathResolver.relativize(baseDir, file); if (relativePathToBaseDir.isEmpty()) { @@ -148,12 +150,13 @@ public class ProjectFileIndexer { } } - private void indexFiles(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, + private void indexFiles(DefaultInputModule module, ModuleConfiguration moduleConfig, ModuleExclusionFilters moduleExclusionFilters, + ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, List<Path> sources, Type type) { try { for (Path dirOrFile : sources) { if (dirOrFile.toFile().isDirectory()) { - indexDirectory(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type); + indexDirectory(module, moduleConfig, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type); } else { fileIndexer.indexFile(module, moduleCoverageAndDuplicationExclusions, dirOrFile, type, progressReport); } @@ -163,18 +166,16 @@ public class ProjectFileIndexer { } } - private void indexDirectory(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, + private void indexDirectory(DefaultInputModule module, ModuleConfiguration moduleConfig, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path dirToIndex, Type type) throws IOException { Files.walkFileTree(dirToIndex.normalize(), Collections.singleton(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, new DirectoryFileVisitor(file -> fileIndexer.indexFile(module, moduleCoverageAndDuplicationExclusions, file, type, progressReport), - module, moduleExclusionFilters, inputModuleHierarchy, type)); + module, moduleConfig, moduleExclusionFilters, inputModuleHierarchy, type, hiddenFilesProjectData)); } private static String pluralizeFiles(int count) { return count == 1 ? "file" : "files"; } - - } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java index 033ab56d3d4..3e7b655589c 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java @@ -66,6 +66,7 @@ public class ProjectFilePreprocessor { private final LanguageDetection languageDetection; private final FilePreprocessor filePreprocessor; private final ProjectExclusionFilters projectExclusionFilters; + private final HiddenFilesProjectData hiddenFilesProjectData; private final SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter; @@ -79,7 +80,7 @@ public class ProjectFilePreprocessor { public ProjectFilePreprocessor(AnalysisWarnings analysisWarnings, ScmConfiguration scmConfiguration, InputModuleHierarchy inputModuleHierarchy, GlobalConfiguration globalConfig, GlobalServerSettings globalServerSettings, ProjectServerSettings projectServerSettings, LanguageDetection languageDetection, FilePreprocessor filePreprocessor, - ProjectExclusionFilters projectExclusionFilters, SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter) { + ProjectExclusionFilters projectExclusionFilters, SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter, HiddenFilesProjectData hiddenFilesProjectData) { this.analysisWarnings = analysisWarnings; this.scmConfiguration = scmConfiguration; this.inputModuleHierarchy = inputModuleHierarchy; @@ -92,6 +93,7 @@ public class ProjectFilePreprocessor { this.sonarGlobalPropertiesFilter = sonarGlobalPropertiesFilter; this.ignoreCommand = loadIgnoreCommand(); this.useScmExclusion = ignoreCommand != null; + this.hiddenFilesProjectData = hiddenFilesProjectData; } public void execute() { @@ -109,7 +111,7 @@ public class ProjectFilePreprocessor { int totalLanguagesDetected = languageDetection.getDetectedLanguages().size(); - progressReport.stop(String.format("%s detected in %s", pluralizeWithCount("language", totalLanguagesDetected), + progressReport.stopAndLogTotalTime(String.format("%s detected in %s", pluralizeWithCount("language", totalLanguagesDetected), pluralizeWithCount("preprocessed file", totalFilesPreprocessed))); int excludedFileByPatternCount = exclusionCounter.getByPatternsCount(); @@ -138,27 +140,31 @@ public class ProjectFilePreprocessor { // Default to index basedir when no sources provided List<Path> mainSourceDirsOrFiles = module.getSourceDirsOrFiles() .orElseGet(() -> hasChildModules || hasTests ? emptyList() : singletonList(module.getBaseDir().toAbsolutePath())); - List<Path> processedSources = processModuleSources(module, moduleExclusionFilters, mainSourceDirsOrFiles, InputFile.Type.MAIN, + List<Path> processedSources = processModuleSources(module, moduleConfig, moduleExclusionFilters, mainSourceDirsOrFiles, InputFile.Type.MAIN, exclusionCounter); mainSourcesByModule.put(module, processedSources); totalFilesPreprocessed += processedSources.size(); module.getTestDirsOrFiles().ifPresent(tests -> { - List<Path> processedTestSources = processModuleSources(module, moduleExclusionFilters, tests, InputFile.Type.TEST, exclusionCounter); + List<Path> processedTestSources = processModuleSources(module, moduleConfig, moduleExclusionFilters, tests, InputFile.Type.TEST, exclusionCounter); testSourcesByModule.put(module, processedTestSources); totalFilesPreprocessed += processedTestSources.size(); }); } - private List<Path> processModuleSources(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, List<Path> sources, + private List<Path> processModuleSources(DefaultInputModule module, ModuleConfiguration moduleConfiguration, ModuleExclusionFilters moduleExclusionFilters, List<Path> sources, InputFile.Type type, ExclusionCounter exclusionCounter) { List<Path> processedFiles = new ArrayList<>(); try { for (Path dirOrFile : sources) { if (dirOrFile.toFile().isDirectory()) { - processedFiles.addAll(processDirectory(module, moduleExclusionFilters, dirOrFile, type, exclusionCounter)); + processedFiles.addAll(processDirectory(module, moduleConfiguration, moduleExclusionFilters, dirOrFile, type, exclusionCounter)); } else { filePreprocessor.processFile(module, moduleExclusionFilters, dirOrFile, type, exclusionCounter, ignoreCommand) - .ifPresent(processedFiles::add); + .ifPresentOrElse( + processedFiles::add, + // If the file is not processed, we don't need to save visibility data and can remove it + () -> hiddenFilesProjectData.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(dirOrFile, module) + ); } } } catch (IOException e) { @@ -167,12 +173,17 @@ public class ProjectFilePreprocessor { return processedFiles; } - private List<Path> processDirectory(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, Path path, + private List<Path> processDirectory(DefaultInputModule module, ModuleConfiguration moduleConfiguration, ModuleExclusionFilters moduleExclusionFilters, Path path, InputFile.Type type, ExclusionCounter exclusionCounter) throws IOException { List<Path> processedFiles = new ArrayList<>(); Files.walkFileTree(path.normalize(), Collections.singleton(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, - new DirectoryFileVisitor(file -> filePreprocessor.processFile(module, moduleExclusionFilters, file, type, exclusionCounter, - ignoreCommand).ifPresent(processedFiles::add), module, moduleExclusionFilters, inputModuleHierarchy, type)); + new DirectoryFileVisitor(file -> filePreprocessor + .processFile(module, moduleExclusionFilters, file, type, exclusionCounter, ignoreCommand) + .ifPresentOrElse( + processedFiles::add, + // If the file is not processed, we don't need to save visibility data and can remove it + () -> hiddenFilesProjectData.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(file, module)), + module, moduleConfiguration, moduleExclusionFilters, inputModuleHierarchy, type, hiddenFilesProjectData)); return processedFiles; } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/AbstractSensorWrapper.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/AbstractSensorWrapper.java index a08380cf9d8..10d75a4b3c5 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/AbstractSensorWrapper.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/AbstractSensorWrapper.java @@ -19,11 +19,11 @@ */ package org.sonar.scanner.sensor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.sonar.api.batch.sensor.SensorContext; import org.sonar.api.batch.sensor.internal.DefaultSensorDescriptor; import org.sonar.api.scanner.sensor.ProjectSensor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.sonar.scanner.scan.branch.BranchConfiguration; import org.sonar.scanner.scan.branch.BranchType; import org.sonar.scanner.scan.filesystem.MutableFileSystem; @@ -60,7 +60,12 @@ public abstract class AbstractSensorWrapper<G extends ProjectSensor> { if (sensorIsRestricted) { LOGGER.info("Sensor {} is restricted to changed files only", descriptor.name()); } + boolean allowHiddenFileAnalysis = descriptor.isProcessesHiddenFiles(); + if (allowHiddenFileAnalysis) { + LOGGER.debug("Sensor {} is allowed to analyze hidden files", descriptor.name()); + } fileSystem.setRestrictToChangedFiles(sensorIsRestricted); + fileSystem.setAllowHiddenFileAnalysis(allowHiddenFileAnalysis); wrappedSensor.execute(context); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/DefaultSensorStorage.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/DefaultSensorStorage.java index 5fa6e33aac6..0ff5109124d 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/DefaultSensorStorage.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/DefaultSensorStorage.java @@ -19,6 +19,16 @@ */ package org.sonar.scanner.sensor; +import static java.lang.Math.max; +import static org.sonar.api.measures.CoreMetrics.COMMENT_LINES_DATA_KEY; +import static org.sonar.api.measures.CoreMetrics.LINES_KEY; +import static org.sonar.api.measures.CoreMetrics.PUBLIC_DOCUMENTED_API_DENSITY_KEY; +import static org.sonar.api.measures.CoreMetrics.TEST_SUCCESS_DENSITY_KEY; +import static org.sonar.api.utils.Preconditions.checkArgument; + +import com.google.protobuf.ByteString; +import java.io.IOException; +import java.io.InputStream; import java.io.Serializable; import java.util.HashSet; import java.util.List; @@ -28,6 +38,7 @@ import java.util.SortedMap; import java.util.TreeMap; import java.util.stream.Collectors; import javax.annotation.Nullable; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.api.batch.fs.InputComponent; @@ -80,12 +91,6 @@ import org.sonar.scanner.repository.ContextPropertiesCache; import org.sonar.scanner.repository.TelemetryCache; import org.sonar.scanner.scan.branch.BranchConfiguration; -import static java.lang.Math.max; -import static org.sonar.api.measures.CoreMetrics.COMMENT_LINES_DATA_KEY; -import static org.sonar.api.measures.CoreMetrics.LINES_KEY; -import static org.sonar.api.measures.CoreMetrics.PUBLIC_DOCUMENTED_API_DENSITY_KEY; -import static org.sonar.api.measures.CoreMetrics.TEST_SUCCESS_DENSITY_KEY; - public class DefaultSensorStorage implements SensorStorage { private static final Logger LOG = LoggerFactory.getLogger(DefaultSensorStorage.class); @@ -122,6 +127,7 @@ public class DefaultSensorStorage implements SensorStorage { private final ScannerMetrics scannerMetrics; private final BranchConfiguration branchConfiguration; private final Set<String> alreadyLogged = new HashSet<>(); + private final Set<String> alreadyAddedData = new HashSet<>(); public DefaultSensorStorage(MetricFinder metricFinder, IssuePublisher moduleIssues, Configuration settings, ReportPublisher reportPublisher, SonarCpdBlockIndex index, ContextPropertiesCache contextPropertiesCache, TelemetryCache telemetryCache, ScannerMetrics scannerMetrics, BranchConfiguration branchConfiguration) { @@ -472,4 +478,23 @@ public class DefaultSensorStorage implements SensorStorage { writer.writeComponentSignificantCode(componentRef, protobuf); } + + public void storeAnalysisData(String key, String mimeType, InputStream data) { + checkArgument(!StringUtils.isBlank(key), "Key must not be null"); + checkArgument(!alreadyAddedData.contains(key), "A data with this key already exists"); + checkArgument(!StringUtils.isBlank(mimeType), "MimeType must not be null"); + checkArgument(data != null, "Data must not be null"); + alreadyAddedData.add(key); + try (data) { + ScannerReport.AnalysisData analysisData = ScannerReport.AnalysisData.newBuilder() + .setKey(key) + .setMimeType(mimeType) + .setData(ByteString.readFrom(data)) + .build(); + ScannerReportWriter writer = reportPublisher.getWriter(); + writer.appendAnalysisData(analysisData); + } catch (IOException e) { + throw new IllegalArgumentException("Failed to read data InputStream", e); + } + } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java index 5f28e7e283e..01b6c0c11cd 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java @@ -28,7 +28,6 @@ import org.sonar.api.batch.rule.ActiveRules; import org.sonar.api.batch.sensor.cache.ReadCache; import org.sonar.api.batch.sensor.cache.WriteCache; import org.sonar.api.config.Configuration; -import org.sonar.api.config.Settings; import org.sonar.scanner.bootstrap.ScannerPluginRepository; import org.sonar.scanner.cache.AnalysisCacheEnabled; import org.sonar.scanner.scan.branch.BranchConfiguration; @@ -38,11 +37,11 @@ public class ModuleSensorContext extends ProjectSensorContext { private final InputModule module; - public ModuleSensorContext(DefaultInputProject project, InputModule module, Configuration config, Settings mutableModuleSettings, FileSystem fs, ActiveRules activeRules, + public ModuleSensorContext(DefaultInputProject project, InputModule module, Configuration config, FileSystem fs, ActiveRules activeRules, DefaultSensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration, WriteCache writeCache, ReadCache readCache, AnalysisCacheEnabled analysisCacheEnabled, UnchangedFilesHandler unchangedFilesHandler, ExecutingSensorContext executingSensorContext, ScannerPluginRepository pluginRepository) { - super(project, config, mutableModuleSettings, fs, activeRules, sensorStorage, sonarRuntime, branchConfiguration, writeCache, readCache, analysisCacheEnabled, + super(project, config, fs, activeRules, sensorStorage, sonarRuntime, branchConfiguration, writeCache, readCache, analysisCacheEnabled, unchangedFilesHandler, executingSensorContext, pluginRepository); this.module = module; } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java index bac06a38645..54c86750eaf 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java @@ -19,6 +19,7 @@ */ package org.sonar.scanner.sensor; +import java.io.InputStream; import java.io.Serializable; import javax.annotation.concurrent.ThreadSafe; import org.sonar.api.SonarRuntime; @@ -65,7 +66,6 @@ public class ProjectSensorContext implements SensorContext { static final NoOpNewAnalysisError NO_OP_NEW_ANALYSIS_ERROR = new NoOpNewAnalysisError(); - private final Settings mutableSettings; private final FileSystem fs; private final ActiveRules activeRules; private final DefaultSensorStorage sensorStorage; @@ -80,15 +80,14 @@ public class ProjectSensorContext implements SensorContext { private final ExecutingSensorContext executingSensorContext; private final ScannerPluginRepository pluginRepo; - public ProjectSensorContext(DefaultInputProject project, Configuration config, Settings mutableSettings, FileSystem fs, - ActiveRules activeRules, - DefaultSensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration, - WriteCache writeCache, ReadCache readCache, - AnalysisCacheEnabled analysisCacheEnabled, UnchangedFilesHandler unchangedFilesHandler, - ExecutingSensorContext executingSensorContext, ScannerPluginRepository pluginRepo) { + public ProjectSensorContext(DefaultInputProject project, Configuration config, FileSystem fs, + ActiveRules activeRules, + DefaultSensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration, + WriteCache writeCache, ReadCache readCache, + AnalysisCacheEnabled analysisCacheEnabled, UnchangedFilesHandler unchangedFilesHandler, + ExecutingSensorContext executingSensorContext, ScannerPluginRepository pluginRepo) { this.project = project; this.config = config; - this.mutableSettings = mutableSettings; this.fs = fs; this.activeRules = activeRules; this.sensorStorage = sensorStorage; @@ -104,7 +103,7 @@ public class ProjectSensorContext implements SensorContext { @Override public Settings settings() { - return mutableSettings; + throw new UnsupportedOperationException("This method is not supported anymore"); } @Override @@ -233,6 +232,15 @@ public class ProjectSensorContext implements SensorContext { } @Override + public void addAnalysisData(String key, String mimeType, InputStream data) { + if (isSonarSourcePlugin()) { + this.sensorStorage.storeAnalysisData(key, mimeType, data); + } else { + throw new IllegalStateException("Analysis data can only be added by SonarSource plugins"); + } + } + + @Override public NewSignificantCode newSignificantCode() { return new DefaultSignificantCode(sensorStorage); } @@ -250,4 +258,5 @@ public class ProjectSensorContext implements SensorContext { } return false; } + } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/CompositeBlameCommand.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/CompositeBlameCommand.java index 0742740bba6..a481f4a54f4 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/CompositeBlameCommand.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/CompositeBlameCommand.java @@ -22,7 +22,9 @@ package org.sonar.scm.git; import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.time.Instant; import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -32,6 +34,7 @@ import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.diff.RawTextComparator; @@ -238,7 +241,7 @@ public class CompositeBlameCommand extends BlameCommand { break; } linesList.add(new BlameLine() - .date(fileBlame.getCommitDates()[i]) + .date(toDate(fileBlame.getCommitDates()[i])) .revision(fileBlame.getCommitHashes()[i]) .author(fileBlame.getAuthorEmails()[i])); } @@ -251,4 +254,8 @@ public class CompositeBlameCommand extends BlameCommand { } } + private static @Nullable Date toDate(@Nullable Instant commitDate) { + return commitDate != null ? Date.from(commitDate) : null; + } + } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmSupport.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmSupport.java index 162e7f71eff..d8aef57bc2f 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmSupport.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmSupport.java @@ -22,6 +22,7 @@ package org.sonar.scm.git; import java.util.Arrays; import java.util.List; import org.eclipse.jgit.util.FS; +import org.sonar.core.util.ProcessWrapperFactory; import org.sonar.scm.git.strategy.DefaultBlameStrategy; public final class GitScmSupport { diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/JGitUtils.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/JGitUtils.java index 972a8ce8da3..bc38a55e619 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/JGitUtils.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/JGitUtils.java @@ -21,6 +21,9 @@ package org.sonar.scm.git; import java.io.IOException; import java.nio.file.Path; +import java.util.List; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.Repository; @@ -41,4 +44,27 @@ public class JGitUtils { throw new IllegalStateException("Unable to open Git repository", e); } } + + // Return a list of scm ignored paths relative to the baseDir. + public static List<String> getAllIgnoredPaths(Path baseDir) { + try (Repository repo = buildRepository(baseDir)) { + Path workTreePath = repo.getWorkTree().toPath(); + Path baseDirAbs = baseDir.toAbsolutePath().normalize(); + + try (Git git = new Git(repo)) { + return git.status().call().getIgnoredNotInIndex().stream() + // Convert to absolute path + .map(filePathStr -> workTreePath.resolve(filePathStr).normalize()) + // Exclude any outside of the baseDir + .filter(filePath -> filePath.startsWith(baseDirAbs)) + // Make path relative to the baseDir + .map(baseDir::relativize) + .map(Path::toString) + .sorted() + .toList(); + } catch (GitAPIException e) { + throw new RuntimeException(e); + } + } + } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/NativeGitBlameCommand.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/NativeGitBlameCommand.java index cac4370e09e..8f066727e21 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/NativeGitBlameCommand.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/NativeGitBlameCommand.java @@ -25,6 +25,7 @@ import java.time.Instant; import java.util.Date; import java.util.LinkedList; import java.util.List; +import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -34,6 +35,7 @@ import org.slf4j.LoggerFactory; import org.sonar.api.batch.scm.BlameLine; import org.sonar.api.utils.System2; import org.sonar.api.utils.Version; +import org.sonar.core.util.ProcessWrapperFactory; import org.springframework.beans.factory.annotation.Autowired; import static java.util.Collections.emptyList; @@ -61,6 +63,7 @@ public class NativeGitBlameCommand { private final System2 system; private final ProcessWrapperFactory processWrapperFactory; + private final Consumer<String> stderrConsumer = line -> LOG.debug("[stderr] {}", line); private String gitCommand; @Autowired @@ -84,7 +87,7 @@ public class NativeGitBlameCommand { try { this.gitCommand = locateDefaultGit(); MutableString stdOut = new MutableString(); - this.processWrapperFactory.create(null, l -> stdOut.string = l, gitCommand, "--version").execute(); + this.processWrapperFactory.create(null, l -> stdOut.string = l, stderrConsumer, gitCommand, "--version").execute(); return stdOut.string != null && stdOut.string.startsWith("git version") && isCompatibleGitVersion(stdOut.string); } catch (Exception e) { LOG.debug("Failed to find git native client", e); @@ -108,7 +111,7 @@ public class NativeGitBlameCommand { // To avoid it we use where.exe to find git binary only in PATH. LOG.debug("Looking for git command in the PATH using where.exe (Windows)"); List<String> whereCommandResult = new LinkedList<>(); - this.processWrapperFactory.create(null, whereCommandResult::add, "C:\\Windows\\System32\\where.exe", "$PATH:git.exe") + this.processWrapperFactory.create(null, whereCommandResult::add, stderrConsumer, "C:\\Windows\\System32\\where.exe", "$PATH:git.exe") .execute(); if (!whereCommandResult.isEmpty()) { @@ -119,18 +122,19 @@ public class NativeGitBlameCommand { throw new IllegalStateException("git.exe not found in PATH. PATH value was: " + system.property("PATH")); } - public List<BlameLine> blame(Path baseDir, String fileName) throws Exception { + public List<BlameLine> blame(Path baseDir, String fileName) throws IOException { BlameOutputProcessor outputProcessor = new BlameOutputProcessor(); - try { - this.processWrapperFactory.create( - baseDir, - outputProcessor::process, - gitCommand, - GIT_DIR_FLAG, String.format(GIT_DIR_ARGUMENT, baseDir), GIT_DIR_FORCE_FLAG, baseDir.toString(), - BLAME_COMMAND, - BLAME_LINE_PORCELAIN_FLAG, IGNORE_WHITESPACES, FILENAME_SEPARATOR_FLAG, fileName) - .execute(); - } catch (UncommittedLineException e) { + var processWrapper = this.processWrapperFactory.create( + baseDir, + outputProcessor::process, + stderrConsumer, + gitCommand, + GIT_DIR_FLAG, String.format(GIT_DIR_ARGUMENT, baseDir), GIT_DIR_FORCE_FLAG, baseDir.toString(), + BLAME_COMMAND, + BLAME_LINE_PORCELAIN_FLAG, IGNORE_WHITESPACES, FILENAME_SEPARATOR_FLAG, fileName); + outputProcessor.setProcessWrapper(processWrapper); + processWrapper.execute(); + if (outputProcessor.hasEncounteredUncommittedLine()) { LOG.debug("Unable to blame file '{}' - it has uncommitted changes", fileName); return emptyList(); } @@ -142,6 +146,8 @@ public class NativeGitBlameCommand { private String sha1 = null; private String committerTime = null; private String authorMail = null; + private ProcessWrapperFactory.ProcessWrapper processWrapper = null; + private volatile boolean encounteredUncommittedLine = false; public List<BlameLine> getBlameLines() { return blameLines; @@ -160,11 +166,16 @@ public class NativeGitBlameCommand { authorMail = matcher.group(1); } if (authorMail.equals("not.committed.yet")) { - throw new UncommittedLineException(); + encounteredUncommittedLine = true; + processWrapper.destroy(); } } } + public boolean hasEncounteredUncommittedLine() { + return encounteredUncommittedLine; + } + private void saveEntry() { checkState(authorMail != null, "Did not find an author email for an entry"); checkState(committerTime != null, "Did not find a committer time for an entry"); @@ -181,6 +192,10 @@ public class NativeGitBlameCommand { sha1 = null; committerTime = null; } + + public void setProcessWrapper(ProcessWrapperFactory.ProcessWrapper processWrapper) { + this.processWrapper = processWrapper; + } } private static boolean isCompatibleGitVersion(String gitVersionCommandOutput) { @@ -207,7 +222,4 @@ public class NativeGitBlameCommand { String string; } - private static class UncommittedLineException extends RuntimeException { - - } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/ProcessWrapperFactory.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/ProcessWrapperFactory.java deleted file mode 100644 index 9fa97ea1cab..00000000000 --- a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/ProcessWrapperFactory.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2025 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.scm.git; - -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.file.Path; -import java.util.HashMap; -import java.util.Map; -import java.util.Scanner; -import java.util.function.Consumer; -import javax.annotation.Nullable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static java.lang.String.format; -import static java.lang.String.join; -import static java.nio.charset.StandardCharsets.UTF_8; - -public class ProcessWrapperFactory { - private static final Logger LOG = LoggerFactory.getLogger(ProcessWrapperFactory.class); - - public ProcessWrapperFactory() { - // nothing to do - } - - public ProcessWrapper create(@Nullable Path baseDir, Consumer<String> stdOutLineConsumer, String... command) { - return new ProcessWrapper(baseDir, stdOutLineConsumer, Map.of(), command); - } - - public ProcessWrapper create(@Nullable Path baseDir, Consumer<String> stdOutLineConsumer, Map<String, String> envVariables, String... command) { - return new ProcessWrapper(baseDir, stdOutLineConsumer, envVariables, command); - } - - static class ProcessWrapper { - - private final Path baseDir; - private final Consumer<String> stdOutLineConsumer; - private final String[] command; - private final Map<String, String> envVariables = new HashMap<>(); - - ProcessWrapper(@Nullable Path baseDir, Consumer<String> stdOutLineConsumer, Map<String, String> envVariables, String... command) { - this.baseDir = baseDir; - this.stdOutLineConsumer = stdOutLineConsumer; - this.envVariables.putAll(envVariables); - this.command = command; - } - - public void execute() throws IOException { - ProcessBuilder pb = new ProcessBuilder() - .command(command) - .directory(baseDir != null ? baseDir.toFile() : null); - envVariables.forEach(pb.environment()::put); - - Process p = pb.start(); - try { - processInputStream(p.getInputStream(), stdOutLineConsumer); - - processInputStream(p.getErrorStream(), line -> { - if (!line.isBlank()) { - LOG.debug(line); - } - }); - - int exit = p.waitFor(); - if (exit != 0) { - throw new IllegalStateException(format("Command execution exited with code: %d", exit)); - } - } catch (InterruptedException e) { - LOG.warn(format("Command [%s] interrupted", join(" ", command)), e); - Thread.currentThread().interrupt(); - } finally { - p.destroy(); - } - } - - private static void processInputStream(InputStream inputStream, Consumer<String> stringConsumer) { - try (Scanner scanner = new Scanner(new InputStreamReader(inputStream, UTF_8))) { - scanner.useDelimiter("\n"); - while (scanner.hasNext()) { - stringConsumer.accept(scanner.next()); - } - } - } - } - -} |