diff options
Diffstat (limited to 'sonar-scanner-engine/src/main/java/org/sonar')
28 files changed, 507 insertions, 287 deletions
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/JGitCleanupService.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/JGitCleanupService.java new file mode 100644 index 00000000000..28e052cfa4e --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/JGitCleanupService.java @@ -0,0 +1,47 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.bootstrap; + +import java.lang.reflect.Method; +import org.eclipse.jgit.internal.util.CleanupService; + +/** + * Normally, JGit terminates with a shutdown hook. Since we also want to support running the Scanner Engine in the same JVM, this allows triggering shutdown manually. + */ +class JGitCleanupService implements AutoCloseable { + + private final Method shutDownMethod; + private final CleanupService cleanupService; + + public JGitCleanupService() { + cleanupService = new CleanupService(); + try { + shutDownMethod = CleanupService.class.getDeclaredMethod("shutDown"); + } catch (NoSuchMethodException e) { + throw new IllegalStateException("Unable to find method 'shutDown' on JGit CleanupService", e); + } + shutDownMethod.setAccessible(true); + } + + @Override + public void close() throws Exception { + shutDownMethod.invoke(cleanupService); + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerMain.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerMain.java index 0fe2c3ad479..bd8d5b9b99c 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerMain.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerMain.java @@ -21,17 +21,21 @@ package org.sonar.scanner.bootstrap; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.OutputStreamAppender; import com.google.gson.Gson; import com.google.gson.annotations.SerializedName; import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import javax.annotation.CheckForNull; +import javax.annotation.Nullable; import org.jetbrains.annotations.NotNull; import org.slf4j.LoggerFactory; import org.sonar.api.utils.MessageException; @@ -49,11 +53,13 @@ public class ScannerMain { private static final String SCANNER_APP_VERSION_KEY = "sonar.scanner.appVersion"; public static void main(String... args) { - System.exit(run(System.in)); + System.exit(run(System.in, System.out)); } - public static int run(InputStream in) { - try { + public static int run(InputStream in, OutputStream out) { + try (var ignored = new JGitCleanupService()) { + configureLogOutput(out); + LOG.info("Starting SonarScanner Engine..."); LOG.atInfo().log(ScannerMain::java); @@ -67,9 +73,11 @@ public class ScannerMain { LOG.info("SonarScanner Engine completed successfully"); return 0; - } catch (Exception e) { - handleException(e); + } catch (Throwable throwable) { + handleException(throwable); return 1; + } finally { + stopLogback(); } } @@ -87,30 +95,28 @@ public class ScannerMain { return sb.toString(); } - private static void handleException(Exception e) { - var messageException = unwrapMessageException(e); + private static void handleException(Throwable throwable) { + var messageException = unwrapMessageException(throwable); if (messageException.isPresent()) { // Don't show the stacktrace for a message exception to not pollute the logs if (LoggerFactory.getLogger(ScannerMain.class).isDebugEnabled()) { - LOG.error(messageException.get(), e); + LOG.error(messageException.get(), throwable); } else { LOG.error(messageException.get()); } } else { - LOG.error("Error during SonarScanner Engine execution", e); + LOG.error("Error during SonarScanner Engine execution", throwable); } } - private static Optional<String> unwrapMessageException(Exception t) { - Throwable y = t; - do { - if (y instanceof MessageException messageException) { - return Optional.of(messageException.getMessage()); - } - y = y.getCause(); - } while (y != null); - - return Optional.empty(); + private static Optional<String> unwrapMessageException(@Nullable Throwable throwable) { + if (throwable == null) { + return Optional.empty(); + } else if (throwable instanceof MessageException messageException) { + return Optional.of(messageException.getMessage()); + } else { + return unwrapMessageException(throwable.getCause()); + } } private static @NotNull Map<String, String> parseInputProperties(InputStream in) { @@ -157,6 +163,28 @@ public class ScannerMain { rootLogger.setLevel(Level.toLevel(verbose ? LEVEL_ROOT_VERBOSE : LEVEL_ROOT_DEFAULT)); } + private static void configureLogOutput(OutputStream out) { + var loggerContext = (ch.qos.logback.classic.LoggerContext) LoggerFactory.getILoggerFactory(); + var encoder = new ScannerLogbackEncoder(); + encoder.setContext(loggerContext); + encoder.start(); + + var appender = new OutputStreamAppender<ILoggingEvent>(); + appender.setEncoder(encoder); + appender.setContext(loggerContext); + appender.setOutputStream(out); + appender.start(); + + var rootLogger = (Logger) LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME); + rootLogger.addAppender(appender); + rootLogger.setLevel(Level.toLevel(LEVEL_ROOT_DEFAULT)); + } + + private static void stopLogback() { + var loggerContext = (ch.qos.logback.classic.LoggerContext) LoggerFactory.getILoggerFactory(); + loggerContext.stop(); + } + private static class Input { @SerializedName("scannerProperties") private List<ScannerProperty> scannerProperties; diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java index 9c41891dbab..133f4387856 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java @@ -99,7 +99,6 @@ import org.sonar.scanner.scan.InputModuleHierarchyProvider; import org.sonar.scanner.scan.InputProjectProvider; import org.sonar.scanner.scan.ModuleIndexer; import org.sonar.scanner.scan.MutableProjectReactorProvider; -import org.sonar.scanner.scan.MutableProjectSettings; import org.sonar.scanner.scan.ProjectBuildersExecutor; import org.sonar.scanner.scan.ProjectConfigurationProvider; import org.sonar.scanner.scan.ProjectLock; @@ -116,6 +115,7 @@ import org.sonar.scanner.scan.branch.BranchType; import org.sonar.scanner.scan.branch.ProjectBranchesProvider; import org.sonar.scanner.scan.filesystem.DefaultProjectFileSystem; import org.sonar.scanner.scan.filesystem.FilePreprocessor; +import org.sonar.scanner.scan.filesystem.HiddenFilesProjectData; import org.sonar.scanner.scan.filesystem.InputComponentStore; import org.sonar.scanner.scan.filesystem.LanguageDetection; import org.sonar.scanner.scan.filesystem.MetadataGenerator; @@ -200,6 +200,7 @@ public class SpringScannerContainer extends SpringComponentContainer { FilePreprocessor.class, ProjectFilePreprocessor.class, ProjectExclusionFilters.class, + HiddenFilesProjectData.class, // rules new ActiveRulesProvider(), @@ -226,7 +227,6 @@ public class SpringScannerContainer extends SpringComponentContainer { ContextPropertiesCache.class, TelemetryCache.class, - MutableProjectSettings.class, SonarGlobalPropertiesFilter.class, ProjectConfigurationProvider.class, diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RulesSeverityDetector.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RulesSeverityDetector.java index 703cd038fd0..d4f22ad9704 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RulesSeverityDetector.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RulesSeverityDetector.java @@ -97,8 +97,11 @@ public class RulesSeverityDetector { } private static Map<String, Result.Level> getDriverDefinedRuleSeverities(Run run) { - return run.getTool().getDriver().getRules() - .stream() + Set<ReportingDescriptor> rules = run.getTool().getDriver().getRules(); + if (rules == null) { + return emptyMap(); + } + return rules.stream() .filter(RulesSeverityDetector::hasRuleDefinedLevel) .collect(toMap(ReportingDescriptor::getId, x -> Result.Level.valueOf(x.getDefaultConfiguration().getLevel().name()))); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RunMapper.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RunMapper.java index 5b9abf383cf..bdf5a9a1114 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RunMapper.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RunMapper.java @@ -83,7 +83,7 @@ public class RunMapper { private List<NewAdHocRule> toNewAdHocRules(Run run, String driverName, Map<String, Result.Level> ruleSeveritiesByRuleId, Map<String, Result.Level> ruleSeveritiesByRuleIdForNewCCT) { - Set<ReportingDescriptor> driverRules = run.getTool().getDriver().getRules(); + Set<ReportingDescriptor> driverRules = Optional.ofNullable(run.getTool().getDriver().getRules()).orElse(Set.of()); Set<ReportingDescriptor> extensionRules = hasExtensions(run.getTool()) ? run.getTool().getExtensions().stream().filter(RunMapper::hasRules).flatMap(extension -> extension.getRules().stream()).collect(toSet()) : Set.of(); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/http/ScannerWsClientProvider.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/http/ScannerWsClientProvider.java index 088ebfb0052..d9eed8bedc8 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/http/ScannerWsClientProvider.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/http/ScannerWsClientProvider.java @@ -87,7 +87,7 @@ public class ScannerWsClientProvider { String responseTimeout = defaultIfBlank(scannerProps.property(SONAR_SCANNER_RESPONSE_TIMEOUT), valueOf(DEFAULT_RESPONSE_TIMEOUT)); String envVarToken = defaultIfBlank(system.envVariable(TOKEN_ENV_VARIABLE), null); String token = defaultIfBlank(scannerProps.property(TOKEN_PROPERTY), envVarToken); - String login = defaultIfBlank(scannerProps.property(CoreProperties.LOGIN), token); + String login = defaultIfBlank(token, scannerProps.property(CoreProperties.LOGIN)); boolean skipSystemTrustMaterial = Boolean.parseBoolean(defaultIfBlank(scannerProps.property(SKIP_SYSTEM_TRUST_MATERIAL), "false")); var sslContext = configureSsl(parseSslConfig(scannerProps, sonarUserHome), system, skipSystemTrustMaterial); connectorBuilder diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliCacheService.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliCacheService.java index 51bb79c9d23..24db0ddec64 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliCacheService.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliCacheService.java @@ -25,6 +25,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.Reader; +import java.io.UncheckedIOException; import java.lang.reflect.Type; import java.nio.file.Files; import java.nio.file.Path; @@ -139,7 +140,7 @@ public class CliCacheService { String checksum = metadataResponse.sha256(); // If we have a matching checksum dir with the existing CLI file, then we are up to date. if (!cachedCliFile(checksum).exists()) { - LOG.debug("CLI checksum mismatch"); + LOG.debug("SCA CLI update detected"); downloadCli(metadataResponse.id(), checksum); telemetryCache.put("scanner.sca.get.cli.cache.hit", "false"); } else { @@ -175,8 +176,8 @@ public class CliCacheService { }.getType(); return new Gson().fromJson(reader, listOfMetadata); } - } catch (Exception e) { - throw new IllegalStateException("Unable to load CLI metadata", e); + } catch (IOException e) { + throw new UncheckedIOException(e); } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliService.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliService.java index 8a8e90cce25..6b3418a8f6a 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliService.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliService.java @@ -29,20 +29,22 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.function.Consumer; +import java.util.stream.Stream; import javax.annotation.Nullable; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVPrinter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.event.Level; +import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DefaultInputModule; import org.sonar.api.platform.Server; import org.sonar.api.utils.System2; import org.sonar.core.util.ProcessWrapperFactory; import org.sonar.scanner.config.DefaultConfiguration; import org.sonar.scanner.repository.TelemetryCache; +import org.sonar.scanner.scan.filesystem.ProjectExclusionFilters; import org.sonar.scanner.scm.ScmConfiguration; import org.sonar.scm.git.JGitUtils; @@ -54,37 +56,42 @@ import org.sonar.scm.git.JGitUtils; */ public class CliService { private static final Logger LOG = LoggerFactory.getLogger(CliService.class); - public static final String EXCLUDED_MANIFESTS_PROP_KEY = "sonar.sca.excludedManifests"; + public static final String SCA_EXCLUSIONS_KEY = "sonar.sca.exclusions"; + public static final String LEGACY_SCA_EXCLUSIONS_KEY = "sonar.sca.excludedManifests"; private final ProcessWrapperFactory processWrapperFactory; private final TelemetryCache telemetryCache; private final System2 system2; private final Server server; private final ScmConfiguration scmConfiguration; + private final ProjectExclusionFilters projectExclusionFilters; - public CliService(ProcessWrapperFactory processWrapperFactory, TelemetryCache telemetryCache, System2 system2, Server server, ScmConfiguration scmConfiguration) { + public CliService(ProcessWrapperFactory processWrapperFactory, TelemetryCache telemetryCache, System2 system2, Server server, ScmConfiguration scmConfiguration, + ProjectExclusionFilters projectExclusionFilters) { this.processWrapperFactory = processWrapperFactory; this.telemetryCache = telemetryCache; this.system2 = system2; this.server = server; this.scmConfiguration = scmConfiguration; + this.projectExclusionFilters = projectExclusionFilters; } - public File generateManifestsZip(DefaultInputModule module, File cliExecutable, DefaultConfiguration configuration) throws IOException, IllegalStateException { + public File generateManifestsArchive(DefaultInputModule module, File cliExecutable, DefaultConfiguration configuration) throws IOException, IllegalStateException { long startTime = system2.now(); boolean success = false; try { - String zipName = "dependency-files.zip"; - Path zipPath = module.getWorkDir().resolve(zipName); + String archiveName = "dependency-files.tar.xz"; + Path archivePath = module.getWorkDir().resolve(archiveName); List<String> args = new ArrayList<>(); args.add(cliExecutable.getAbsolutePath()); args.add("projects"); args.add("save-lockfiles"); - args.add("--zip"); - args.add("--zip-filename"); - args.add(zipPath.toAbsolutePath().toString()); + args.add("--xz"); + args.add("--xz-filename"); + args.add(archivePath.toAbsolutePath().toString()); args.add("--directory"); args.add(module.getBaseDir().toString()); + args.add("--recursive"); String excludeFlag = getExcludeFlag(module, configuration); if (excludeFlag != null) { @@ -92,8 +99,7 @@ public class CliService { args.add(excludeFlag); } - boolean scaDebug = configuration.getBoolean("sonar.sca.debug").orElse(false); - if (LOG.isDebugEnabled() || scaDebug) { + if (LOG.isDebugEnabled()) { LOG.info("Setting CLI to debug mode"); args.add("--debug"); } @@ -104,18 +110,16 @@ public class CliService { envProperties.put("TIDELIFT_ALLOW_MANIFEST_FAILURES", "1"); envProperties.put("TIDELIFT_CLI_INSIDE_SCANNER_ENGINE", "1"); envProperties.put("TIDELIFT_CLI_SQ_SERVER_VERSION", server.getVersion()); - // EXCLUDED_MANIFESTS_PROP_KEY is a special case which we handle via --args, not environment variables - Set<String> ignoredProperties = Set.of(EXCLUDED_MANIFESTS_PROP_KEY); - envProperties.putAll(ScaProperties.buildFromScannerProperties(configuration, ignoredProperties)); + envProperties.putAll(ScaProperties.buildFromScannerProperties(configuration)); LOG.info("Running command: {}", args); LOG.info("Environment properties: {}", envProperties); Consumer<String> logConsumer = LOG.atLevel(Level.INFO)::log; processWrapperFactory.create(module.getWorkDir(), logConsumer, logConsumer, envProperties, args.toArray(new String[0])).execute(); - LOG.info("Generated manifests zip file: {}", zipName); + LOG.info("Generated manifests archive file: {}", archiveName); success = true; - return zipPath.toFile(); + return archivePath.toFile(); } finally { telemetryCache.put("scanner.sca.execution.cli.duration", String.valueOf(system2.now() - startTime)); telemetryCache.put("scanner.sca.execution.cli.success", String.valueOf(success)); @@ -123,7 +127,7 @@ public class CliService { } private @Nullable String getExcludeFlag(DefaultInputModule module, DefaultConfiguration configuration) throws IOException { - List<String> configExcludedPaths = getConfigExcludedPaths(configuration); + List<String> configExcludedPaths = getConfigExcludedPaths(configuration, projectExclusionFilters); List<String> scmIgnoredPaths = getScmIgnoredPaths(module); ArrayList<String> mergedExclusionPaths = new ArrayList<>(); @@ -143,12 +147,15 @@ public class CliService { return toCsvString(mergedExclusionPaths); } - private static List<String> getConfigExcludedPaths(DefaultConfiguration configuration) { - String[] excludedPaths = configuration.getStringArray(EXCLUDED_MANIFESTS_PROP_KEY); - if (excludedPaths == null) { - return List.of(); - } - return Arrays.stream(excludedPaths).toList(); + private static List<String> getConfigExcludedPaths(DefaultConfiguration configuration, ProjectExclusionFilters projectExclusionFilters) { + String[] sonarExclusions = projectExclusionFilters.getExclusionsConfig(InputFile.Type.MAIN); + String[] scaExclusions = configuration.getStringArray(SCA_EXCLUSIONS_KEY); + String[] scaExclusionsLegacy = configuration.getStringArray(LEGACY_SCA_EXCLUSIONS_KEY); + + return Stream.of(sonarExclusions, scaExclusions, scaExclusionsLegacy) + .flatMap(Arrays::stream) + .distinct() + .toList(); } private List<String> getScmIgnoredPaths(DefaultInputModule module) { @@ -170,7 +177,13 @@ public class CliService { } return scmIgnoredPaths.stream() .map(ignoredPathRel -> { - boolean isDirectory = Files.isDirectory(baseDirPath.resolve(ignoredPathRel)); + + boolean isDirectory = false; + try { + isDirectory = Files.isDirectory(baseDirPath.resolve(ignoredPathRel.replace("/", File.separator))); + } catch (java.nio.file.InvalidPathException e) { + // if it's not a valid path, it's not a directory so we can just pass to the Tidelift CLI + } // Directories need to get turned into a glob for the Tidelift CLI return isDirectory ? (ignoredPathRel + "/**") : ignoredPathRel; }) diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaExecutor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaExecutor.java index 06142fadb8f..143e144c2dc 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaExecutor.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaExecutor.java @@ -21,6 +21,8 @@ package org.sonar.scanner.sca; import java.io.File; import java.io.IOException; +import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.time.StopWatch; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.api.batch.fs.internal.DefaultInputModule; @@ -65,18 +67,25 @@ public class ScaExecutor { return; } + var stopwatch = new StopWatch(); + stopwatch.start(); LOG.info("Checking for latest CLI"); File cliFile = cliCacheService.cacheCli(); LOG.info("Collecting manifests for the dependency analysis..."); if (cliFile.exists()) { try { - File generatedZip = cliService.generateManifestsZip(root, cliFile, configuration); + File generatedZip = cliService.generateManifestsArchive(root, cliFile, configuration); LOG.debug("Zip ready for report: {}", generatedZip); reportPublisher.getWriter().writeScaFile(generatedZip); LOG.debug("Manifest zip written to report"); } catch (IOException | IllegalStateException e) { LOG.error("Error gathering manifests", e); + } finally { + stopwatch.stop(); + if (LOG.isInfoEnabled()) { + LOG.info("Load SCA project dependencies (done) | time={}ms", stopwatch.getTime(TimeUnit.MILLISECONDS)); + } } } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaProperties.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaProperties.java index 5c848b4ddbc..a697aef3e20 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaProperties.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaProperties.java @@ -30,6 +30,13 @@ import org.sonar.scanner.config.DefaultConfiguration; public class ScaProperties { private static final Pattern sonarScaPropertyRegex = Pattern.compile("^sonar\\.sca\\.([a-zA-Z]+)$"); private static final String SONAR_SCA_PREFIX = "sonar.sca."; + private static final Set<String> IGNORED_PROPERTIES = Set.of( + // sonar.sca.exclusions is a special case which we handle when building --exclude + "sonar.sca.exclusions", + // excludedManifests is a special case which we handle when building --exclude + "sonar.sca.excludedManifests", + // keep recursive enabled to better match sonar-scanner behavior + "sonar.sca.recursiveManifestSearch"); private ScaProperties() { } @@ -46,22 +53,16 @@ public class ScaProperties { * { "sonar.someOtherProperty" : "value" } returns an empty map * * @param configuration the scanner configuration possibly containing sonar.sca.* properties - * @param ignoredPropertyNames property names that should not be processed as a property * @return a map of Tidelift CLI compatible environment variable names to their configuration values */ - public static Map<String, String> buildFromScannerProperties(DefaultConfiguration configuration, Set<String> ignoredPropertyNames) { + public static Map<String, String> buildFromScannerProperties(DefaultConfiguration configuration) { HashMap<String, String> props = new HashMap<>(configuration.getProperties()); - // recursive mode defaults to true - if (!props.containsKey("sonar.sca.recursiveManifestSearch")) { - props.put("sonar.sca.recursiveManifestSearch", "true"); - } - return props .entrySet() .stream() .filter(entry -> entry.getKey().startsWith(SONAR_SCA_PREFIX)) - .filter(entry -> !ignoredPropertyNames.contains(entry.getKey())) + .filter(entry -> !IGNORED_PROPERTIES.contains(entry.getKey())) .collect(Collectors.toMap(entry -> convertPropToEnvVariable(entry.getKey()), Map.Entry::getValue)); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/MutableModuleSettings.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/MutableModuleSettings.java deleted file mode 100644 index 15912f8a510..00000000000 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/MutableModuleSettings.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2025 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.scanner.scan; - -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import jakarta.annotation.Priority; -import org.sonar.api.config.internal.Settings; - -import static java.util.Objects.requireNonNull; - -/** - * @deprecated since 6.5 {@link ModuleConfiguration} used to be mutable, so keep a mutable copy for backward compatibility. - */ -@Deprecated -@Priority(1) -public class MutableModuleSettings extends Settings { - - private final Map<String, String> properties = new HashMap<>(); - - public MutableModuleSettings(ModuleConfiguration config) { - super(config.getDefinitions(), config.getEncryption()); - addProperties(config.getProperties()); - } - - @Override - protected Optional<String> get(String key) { - return Optional.ofNullable(properties.get(key)); - } - - @Override - protected void set(String key, String value) { - properties.put( - requireNonNull(key, "key can't be null"), - requireNonNull(value, "value can't be null").trim()); - } - - @Override - protected void remove(String key) { - properties.remove(key); - } - - @Override - public Map<String, String> getProperties() { - return properties; - } -} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/MutableProjectSettings.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/MutableProjectSettings.java deleted file mode 100644 index df24cbe81e5..00000000000 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/MutableProjectSettings.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2025 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.scanner.scan; - -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import org.sonar.api.config.internal.Settings; -import org.sonar.scanner.bootstrap.GlobalConfiguration; - -import jakarta.annotation.Priority; - -import static java.util.Objects.requireNonNull; - -/** - * @deprecated since 6.5 {@link ProjectConfiguration} used to be mutable, so keep a mutable copy for backward compatibility. - */ -@Deprecated -@Priority(2) -public class MutableProjectSettings extends Settings { - - private final Map<String, String> properties = new HashMap<>(); - - public MutableProjectSettings(GlobalConfiguration globalConfig) { - super(globalConfig.getDefinitions(), globalConfig.getEncryption()); - addProperties(globalConfig.getProperties()); - } - - public void complete(ProjectConfiguration projectConfig) { - addProperties(projectConfig.getProperties()); - } - - @Override - protected Optional<String> get(String key) { - return Optional.ofNullable(properties.get(key)); - } - - @Override - protected void set(String key, String value) { - properties.put( - requireNonNull(key, "key can't be null"), - requireNonNull(value, "value can't be null").trim()); - } - - @Override - protected void remove(String key) { - properties.remove(key); - } - - @Override - public Map<String, String> getProperties() { - return properties; - } -} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectConfigurationProvider.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectConfigurationProvider.java index c12ec245924..e5543d4f9c5 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectConfigurationProvider.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectConfigurationProvider.java @@ -26,7 +26,6 @@ import org.sonar.scanner.bootstrap.GlobalConfiguration; import org.sonar.scanner.bootstrap.GlobalServerSettings; import org.springframework.context.annotation.Bean; - public class ProjectConfigurationProvider { private final SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter; @@ -37,7 +36,7 @@ public class ProjectConfigurationProvider { @Bean("ProjectConfiguration") public ProjectConfiguration provide(DefaultInputProject project, GlobalConfiguration globalConfig, GlobalServerSettings globalServerSettings, - ProjectServerSettings projectServerSettings, MutableProjectSettings projectSettings) { + ProjectServerSettings projectServerSettings) { Map<String, String> settings = new LinkedHashMap<>(); settings.putAll(globalServerSettings.properties()); settings.putAll(projectServerSettings.properties()); @@ -45,10 +44,7 @@ public class ProjectConfigurationProvider { settings = sonarGlobalPropertiesFilter.enforceOnlyServerSideSonarGlobalPropertiesAreUsed(settings, globalServerSettings.properties()); - ProjectConfiguration projectConfig = new ProjectConfiguration(globalConfig.getDefinitions(), globalConfig.getEncryption(), settings); - projectSettings.complete(projectConfig); - return projectConfig; + return new ProjectConfiguration(globalConfig.getDefinitions(), globalConfig.getEncryption(), settings); } - } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringModuleScanContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringModuleScanContainer.java index 4315c762481..8ddb889912d 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringModuleScanContainer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringModuleScanContainer.java @@ -54,7 +54,6 @@ public class SpringModuleScanContainer extends SpringComponentContainer { add( module.definition(), module, - MutableModuleSettings.class, SonarGlobalPropertiesFilter.class, ModuleConfigurationProvider.class, @@ -68,8 +67,7 @@ public class SpringModuleScanContainer extends SpringComponentContainer { ModuleSensorOptimizer.class, ModuleSensorContext.class, - ModuleSensorExtensionDictionary.class - ); + ModuleSensorExtensionDictionary.class); } private void addExtensions() { diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java index 314e923ce71..242bc015574 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java @@ -24,17 +24,15 @@ import java.nio.file.AccessDeniedException; import java.nio.file.FileSystemLoopException; import java.nio.file.FileVisitResult; import java.nio.file.FileVisitor; -import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.attribute.DosFileAttributes; -import org.apache.commons.lang3.SystemUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DefaultInputModule; import org.sonar.scanner.fs.InputModuleHierarchy; +import org.sonar.scanner.scan.ModuleConfiguration; public class DirectoryFileVisitor implements FileVisitor<Path> { @@ -43,27 +41,31 @@ public class DirectoryFileVisitor implements FileVisitor<Path> { private final FileVisitAction fileVisitAction; private final DefaultInputModule module; private final ModuleExclusionFilters moduleExclusionFilters; - private final InputModuleHierarchy inputModuleHierarchy; private final InputFile.Type type; + private final HiddenFilesVisitorHelper hiddenFilesVisitorHelper; - DirectoryFileVisitor(FileVisitAction fileVisitAction, DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, - InputModuleHierarchy inputModuleHierarchy, InputFile.Type type) { + DirectoryFileVisitor(FileVisitAction fileVisitAction, DefaultInputModule module, ModuleConfiguration moduleConfig, ModuleExclusionFilters moduleExclusionFilters, + InputModuleHierarchy inputModuleHierarchy, InputFile.Type type, HiddenFilesProjectData hiddenFilesProjectData) { this.fileVisitAction = fileVisitAction; this.module = module; this.moduleExclusionFilters = moduleExclusionFilters; this.inputModuleHierarchy = inputModuleHierarchy; this.type = type; + this.hiddenFilesVisitorHelper = new HiddenFilesVisitorHelper(hiddenFilesProjectData, module, moduleConfig); } @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - return isHidden(dir) ? FileVisitResult.SKIP_SUBTREE : FileVisitResult.CONTINUE; + if (hiddenFilesVisitorHelper.shouldVisitDir(dir)) { + return FileVisitResult.CONTINUE; + } + return FileVisitResult.SKIP_SUBTREE; } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - if (!Files.isHidden(file)) { + if (hiddenFilesVisitorHelper.shouldVisitFile(file)) { fileVisitAction.execute(file); } return FileVisitResult.CONTINUE; @@ -129,25 +131,12 @@ public class DirectoryFileVisitor implements FileVisitor<Path> { @Override public FileVisitResult postVisitDirectory(Path dir, IOException exc) { + hiddenFilesVisitorHelper.exitDirectory(dir); return FileVisitResult.CONTINUE; } - private static boolean isHidden(Path path) throws IOException { - if (SystemUtils.IS_OS_WINDOWS) { - try { - DosFileAttributes dosFileAttributes = Files.readAttributes(path, DosFileAttributes.class, LinkOption.NOFOLLOW_LINKS); - return dosFileAttributes.isHidden(); - } catch (UnsupportedOperationException e) { - return path.toFile().isHidden(); - } - } else { - return Files.isHidden(path); - } - } - @FunctionalInterface interface FileVisitAction { void execute(Path file) throws IOException; } } - diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java index 7f31c949132..0961edbd985 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java @@ -63,12 +63,13 @@ public class FileIndexer { private final ModuleRelativePathWarner moduleRelativePathWarner; private final InputFileFilterRepository inputFileFilterRepository; private final Languages languages; + private final HiddenFilesProjectData hiddenFilesProjectData; public FileIndexer(DefaultInputProject project, ScannerComponentIdGenerator scannerComponentIdGenerator, InputComponentStore componentStore, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, IssueExclusionsLoader issueExclusionsLoader, MetadataGenerator metadataGenerator, SensorStrategy sensorStrategy, LanguageDetection languageDetection, ScanProperties properties, ScmChangedFiles scmChangedFiles, StatusDetection statusDetection, ModuleRelativePathWarner moduleRelativePathWarner, - InputFileFilterRepository inputFileFilterRepository, Languages languages) { + InputFileFilterRepository inputFileFilterRepository, Languages languages, HiddenFilesProjectData hiddenFilesProjectData) { this.project = project; this.scannerComponentIdGenerator = scannerComponentIdGenerator; this.componentStore = componentStore; @@ -83,15 +84,18 @@ public class FileIndexer { this.moduleRelativePathWarner = moduleRelativePathWarner; this.inputFileFilterRepository = inputFileFilterRepository; this.languages = languages; + this.hiddenFilesProjectData = hiddenFilesProjectData; } - void indexFile(DefaultInputModule module, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path sourceFile, - Type type, ProgressReport progressReport) { + void indexFile(DefaultInputModule module, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path sourceFile, Type type, + ProgressReport progressReport) { Path projectRelativePath = project.getBaseDir().relativize(sourceFile); Path moduleRelativePath = module.getBaseDir().relativize(sourceFile); // This should be fast; language should be cached from preprocessing step Language language = langDetection.language(sourceFile, projectRelativePath); + // cached from directory file visitation, after querying the data is removed to reduce memory consumption + boolean isHidden = hiddenFilesProjectData.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(sourceFile, module); DefaultIndexedFile indexedFile = new DefaultIndexedFile( sourceFile, @@ -102,11 +106,12 @@ public class FileIndexer { language != null ? language.key() : null, scannerComponentIdGenerator.getAsInt(), sensorStrategy, - scmChangedFiles.getOldRelativeFilePath(sourceFile)); + scmChangedFiles.getOldRelativeFilePath(sourceFile), + isHidden); DefaultInputFile inputFile = new DefaultInputFile(indexedFile, f -> metadataGenerator.setMetadata(module.key(), f, module.getEncoding()), f -> f.setStatus(statusDetection.findStatusFromScm(f))); - if (language != null && isPublishAllFiles(language.key())) { + if (!isHidden && language != null && isPublishAllFiles(language.key())) { inputFile.setPublished(true); } if (!accept(inputFile)) { diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java index 544fe46c43b..a87c5f11fc9 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java @@ -147,22 +147,35 @@ public class FilePreprocessor { return true; } - Path target = Files.readSymbolicLink(absolutePath); - if (!Files.exists(target)) { + Optional<Path> target = resolvePathToTarget(absolutePath); + if (target.isEmpty() || !Files.exists(target.get())) { LOG.warn("File '{}' is ignored. It is a symbolic link targeting a file that does not exist.", absolutePath); return false; } - if (!target.startsWith(project.getBaseDir())) { + if (!target.get().startsWith(project.getBaseDir())) { LOG.warn("File '{}' is ignored. It is a symbolic link targeting a file not located in project basedir.", absolutePath); return false; } - if (!target.startsWith(moduleBaseDirectory)) { + if (!target.get().startsWith(moduleBaseDirectory)) { LOG.info("File '{}' is ignored. It is a symbolic link targeting a file not located in module basedir.", absolutePath); return false; } return true; } + + private static Optional<Path> resolvePathToTarget(Path symbolicLinkAbsolutePath) throws IOException { + Path target = Files.readSymbolicLink(symbolicLinkAbsolutePath); + if (target.isAbsolute()) { + return Optional.of(target); + } + + try { + return Optional.of(symbolicLinkAbsolutePath.getParent().resolve(target).toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize()); + } catch (IOException e) { + return Optional.empty(); + } + } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectData.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectData.java new file mode 100644 index 00000000000..d779a054455 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectData.java @@ -0,0 +1,77 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.scan.filesystem; + +import java.io.IOException; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import org.apache.commons.lang3.SystemUtils; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.scanner.bootstrap.SonarUserHome; + +public class HiddenFilesProjectData { + + final Map<DefaultInputModule, Set<Path>> hiddenFilesByModule = new HashMap<>(); + private final SonarUserHome sonarUserHome; + private Path cachedSonarUserHomePath; + + public HiddenFilesProjectData(SonarUserHome sonarUserHome) { + this.sonarUserHome = sonarUserHome; + } + + public void markAsHiddenFile(Path file, DefaultInputModule module) { + hiddenFilesByModule.computeIfAbsent(module, k -> new HashSet<>()).add(file); + } + + /** + * To alleviate additional strain on the memory, we remove the visibility information for <code>hiddenFilesByModule</code> mapdirectly after querying, + * as we don't need it afterward. + */ + public boolean getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(Path file, DefaultInputModule module) { + Set<Path> hiddenFilesPerModule = hiddenFilesByModule.get(module); + if (hiddenFilesPerModule != null) { + return hiddenFilesPerModule.remove(file); + } + return false; + } + + public Path getCachedSonarUserHomePath() throws IOException { + if (cachedSonarUserHomePath == null) { + cachedSonarUserHomePath = resolveRealPath(sonarUserHome.getPath()); + } + return cachedSonarUserHomePath; + } + + public void clearHiddenFilesData() { + // Allowing the GC to collect the map, should only be done after all indexing is complete + hiddenFilesByModule.clear(); + } + + public Path resolveRealPath(Path path) throws IOException { + if (SystemUtils.IS_OS_WINDOWS) { + return path.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize(); + } + return path; + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesVisitorHelper.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesVisitorHelper.java new file mode 100644 index 00000000000..607a859ef44 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesVisitorHelper.java @@ -0,0 +1,112 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.scan.filesystem; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.attribute.DosFileAttributes; +import org.apache.commons.lang3.SystemUtils; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.scanner.scan.ModuleConfiguration; + +public class HiddenFilesVisitorHelper { + + private static final String EXCLUDE_HIDDEN_FILES_PROPERTY = "sonar.scanner.excludeHiddenFiles"; + private final HiddenFilesProjectData hiddenFilesProjectData; + private final DefaultInputModule module; + final boolean excludeHiddenFiles; + private Path moduleWorkDir; + Path rootHiddenDir; + + public HiddenFilesVisitorHelper(HiddenFilesProjectData hiddenFilesProjectData, DefaultInputModule module, ModuleConfiguration moduleConfig) { + this.hiddenFilesProjectData = hiddenFilesProjectData; + this.module = module; + this.excludeHiddenFiles = moduleConfig.getBoolean(EXCLUDE_HIDDEN_FILES_PROPERTY).orElse(false); + } + + public boolean shouldVisitDir(Path path) throws IOException { + boolean isHidden = isHiddenDir(path); + + if (isHidden && (excludeHiddenFiles || isExcludedHiddenDirectory(path))) { + return false; + } + if (isHidden) { + enterHiddenDirectory(path); + } + return true; + } + + private boolean isExcludedHiddenDirectory(Path path) throws IOException { + return getCachedModuleWorkDir().equals(path) || hiddenFilesProjectData.getCachedSonarUserHomePath().equals(path); + } + + void enterHiddenDirectory(Path dir) { + if (!insideHiddenDirectory()) { + rootHiddenDir = dir; + } + } + + public void exitDirectory(Path path) { + if (insideHiddenDirectory() && rootHiddenDir.equals(path)) { + resetRootHiddenDir(); + } + } + + void resetRootHiddenDir() { + this.rootHiddenDir = null; + } + + public boolean shouldVisitFile(Path path) throws IOException { + boolean isHidden = insideHiddenDirectory() || Files.isHidden(path); + + if (!excludeHiddenFiles && isHidden) { + hiddenFilesProjectData.markAsHiddenFile(path, module); + } + + return !excludeHiddenFiles || !isHidden; + } + + private Path getCachedModuleWorkDir() throws IOException { + if (moduleWorkDir == null) { + moduleWorkDir = hiddenFilesProjectData.resolveRealPath(module.getWorkDir()); + } + return moduleWorkDir; + } + + // visible for testing + boolean insideHiddenDirectory() { + return rootHiddenDir != null; + } + + protected static boolean isHiddenDir(Path path) throws IOException { + if (SystemUtils.IS_OS_WINDOWS) { + try { + DosFileAttributes dosFileAttributes = Files.readAttributes(path, DosFileAttributes.class, LinkOption.NOFOLLOW_LINKS); + return dosFileAttributes.isHidden(); + } catch (UnsupportedOperationException e) { + return path.toFile().isHidden(); + } + } else { + return Files.isHidden(path); + } + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStore.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStore.java index 6ef26dafd07..68b6d1db580 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStore.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStore.java @@ -19,12 +19,15 @@ */ package org.sonar.scanner.scan.filesystem; +import java.util.Set; import java.util.SortedSet; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import org.sonar.api.batch.ScannerSide; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.InputModule; -import org.sonar.api.batch.fs.internal.SensorStrategy; import org.sonar.api.batch.fs.internal.DefaultFileSystem; +import org.sonar.api.batch.fs.internal.SensorStrategy; @ScannerSide public class ModuleInputComponentStore extends DefaultFileSystem.Cache { @@ -73,11 +76,29 @@ public class ModuleInputComponentStore extends DefaultFileSystem.Cache { @Override public Iterable<InputFile> getFilesByName(String filename) { - return inputComponentStore.getFilesByName(filename); + Iterable<InputFile> allFilesByName = inputComponentStore.getFilesByName(filename); + if (strategy.isGlobal()) { + return allFilesByName; + } + + return filterByModule(allFilesByName); } @Override public Iterable<InputFile> getFilesByExtension(String extension) { - return inputComponentStore.getFilesByExtension(extension); + Iterable<InputFile> allFilesByExtension = inputComponentStore.getFilesByExtension(extension); + if (strategy.isGlobal()) { + return allFilesByExtension; + } + + return filterByModule(allFilesByExtension); + } + + private Iterable<InputFile> filterByModule(Iterable<InputFile> projectInputFiles) { + Set<InputFile> projectInputFilesSet = StreamSupport.stream(projectInputFiles.spliterator(), false) + .collect(Collectors.toSet()); + return StreamSupport.stream(inputComponentStore.filesByModule(moduleKey).spliterator(), false) + .filter(projectInputFilesSet::contains) + .toList(); } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/MutableFileSystem.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/MutableFileSystem.java index 5daa384d3ac..9c969f6ae20 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/MutableFileSystem.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/MutableFileSystem.java @@ -25,35 +25,54 @@ import org.sonar.api.batch.fs.FilePredicates; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DefaultFileSystem; import org.sonar.api.batch.fs.internal.predicates.ChangedFilePredicate; +import org.sonar.api.batch.fs.internal.predicates.NonHiddenFilesPredicate; public class MutableFileSystem extends DefaultFileSystem { - private boolean restrictToChangedFiles = false; + + boolean restrictToChangedFiles = false; + boolean allowHiddenFileAnalysis = false; public MutableFileSystem(Path baseDir, Cache cache, FilePredicates filePredicates) { super(baseDir, cache, filePredicates); } - public MutableFileSystem(Path baseDir) { + MutableFileSystem(Path baseDir) { super(baseDir); } @Override public Iterable<InputFile> inputFiles(FilePredicate requestPredicate) { - if (restrictToChangedFiles) { - return super.inputFiles(new ChangedFilePredicate(requestPredicate)); - } - return super.inputFiles(requestPredicate); + return super.inputFiles(applyAdditionalPredicate(requestPredicate)); } @Override public InputFile inputFile(FilePredicate requestPredicate) { + return super.inputFile(applyAdditionalPredicate(requestPredicate)); + } + + private FilePredicate applyAdditionalPredicate(FilePredicate requestPredicate) { + return applyHiddenFilePredicate(applyChangedFilePredicate(requestPredicate)); + } + + private FilePredicate applyHiddenFilePredicate(FilePredicate predicate) { + if (allowHiddenFileAnalysis) { + return predicate; + } + return predicates().and(new NonHiddenFilesPredicate(), predicate); + } + + private FilePredicate applyChangedFilePredicate(FilePredicate predicate) { if (restrictToChangedFiles) { - return super.inputFile(new ChangedFilePredicate(requestPredicate)); + return predicates().and(new ChangedFilePredicate(), predicate); } - return super.inputFile(requestPredicate); + return predicate; } public void setRestrictToChangedFiles(boolean restrictToChangedFiles) { this.restrictToChangedFiles = restrictToChangedFiles; } + + public void setAllowHiddenFileAnalysis(boolean allowHiddenFileAnalysis) { + this.allowHiddenFileAnalysis = allowHiddenFileAnalysis; + } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java index 97e449fcb26..c1349872c24 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java @@ -62,6 +62,7 @@ public class ProjectFileIndexer { private final FileIndexer fileIndexer; private final ProjectFilePreprocessor projectFilePreprocessor; private final AnalysisWarnings analysisWarnings; + private final HiddenFilesProjectData hiddenFilesProjectData; private ProgressReport progressReport; @@ -69,7 +70,7 @@ public class ProjectFileIndexer { SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter, InputModuleHierarchy inputModuleHierarchy, GlobalConfiguration globalConfig, GlobalServerSettings globalServerSettings, ProjectServerSettings projectServerSettings, FileIndexer fileIndexer, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, - ProjectFilePreprocessor projectFilePreprocessor, AnalysisWarnings analysisWarnings) { + ProjectFilePreprocessor projectFilePreprocessor, AnalysisWarnings analysisWarnings, HiddenFilesProjectData hiddenFilesProjectData) { this.componentStore = componentStore; this.sonarGlobalPropertiesFilter = sonarGlobalPropertiesFilter; this.inputModuleHierarchy = inputModuleHierarchy; @@ -81,6 +82,7 @@ public class ProjectFileIndexer { this.projectCoverageAndDuplicationExclusions = projectCoverageAndDuplicationExclusions; this.projectFilePreprocessor = projectFilePreprocessor; this.analysisWarnings = analysisWarnings; + this.hiddenFilesProjectData = hiddenFilesProjectData; } public void index() { @@ -91,10 +93,10 @@ public class ProjectFileIndexer { projectCoverageAndDuplicationExclusions.log(" "); indexModulesRecursively(inputModuleHierarchy.root()); + hiddenFilesProjectData.clearHiddenFilesData(); int totalIndexed = componentStore.inputFiles().size(); - progressReport.stop(totalIndexed + " " + pluralizeFiles(totalIndexed) + " indexed"); - + progressReport.stopAndLogTotalTime(totalIndexed + " " + pluralizeFiles(totalIndexed) + " indexed"); } private void indexModulesRecursively(DefaultInputModule module) { @@ -118,15 +120,15 @@ public class ProjectFileIndexer { moduleCoverageAndDuplicationExclusions.log(" "); } List<Path> mainSourceDirsOrFiles = projectFilePreprocessor.getMainSourcesByModule(module); - indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, mainSourceDirsOrFiles, Type.MAIN); + indexFiles(module, moduleConfig, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, mainSourceDirsOrFiles, Type.MAIN); projectFilePreprocessor.getTestSourcesByModule(module) - .ifPresent(tests -> indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, tests, Type.TEST)); + .ifPresent(tests -> indexFiles(module, moduleConfig, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, tests, Type.TEST)); } private static void logPaths(String label, Path baseDir, List<Path> paths) { if (!paths.isEmpty()) { StringBuilder sb = new StringBuilder(label); - for (Iterator<Path> it = paths.iterator(); it.hasNext(); ) { + for (Iterator<Path> it = paths.iterator(); it.hasNext();) { Path file = it.next(); Optional<String> relativePathToBaseDir = PathResolver.relativize(baseDir, file); if (relativePathToBaseDir.isEmpty()) { @@ -148,12 +150,13 @@ public class ProjectFileIndexer { } } - private void indexFiles(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, + private void indexFiles(DefaultInputModule module, ModuleConfiguration moduleConfig, ModuleExclusionFilters moduleExclusionFilters, + ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, List<Path> sources, Type type) { try { for (Path dirOrFile : sources) { if (dirOrFile.toFile().isDirectory()) { - indexDirectory(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type); + indexDirectory(module, moduleConfig, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type); } else { fileIndexer.indexFile(module, moduleCoverageAndDuplicationExclusions, dirOrFile, type, progressReport); } @@ -163,18 +166,16 @@ public class ProjectFileIndexer { } } - private void indexDirectory(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, + private void indexDirectory(DefaultInputModule module, ModuleConfiguration moduleConfig, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path dirToIndex, Type type) throws IOException { Files.walkFileTree(dirToIndex.normalize(), Collections.singleton(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, new DirectoryFileVisitor(file -> fileIndexer.indexFile(module, moduleCoverageAndDuplicationExclusions, file, type, progressReport), - module, moduleExclusionFilters, inputModuleHierarchy, type)); + module, moduleConfig, moduleExclusionFilters, inputModuleHierarchy, type, hiddenFilesProjectData)); } private static String pluralizeFiles(int count) { return count == 1 ? "file" : "files"; } - - } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java index 033ab56d3d4..3e7b655589c 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java @@ -66,6 +66,7 @@ public class ProjectFilePreprocessor { private final LanguageDetection languageDetection; private final FilePreprocessor filePreprocessor; private final ProjectExclusionFilters projectExclusionFilters; + private final HiddenFilesProjectData hiddenFilesProjectData; private final SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter; @@ -79,7 +80,7 @@ public class ProjectFilePreprocessor { public ProjectFilePreprocessor(AnalysisWarnings analysisWarnings, ScmConfiguration scmConfiguration, InputModuleHierarchy inputModuleHierarchy, GlobalConfiguration globalConfig, GlobalServerSettings globalServerSettings, ProjectServerSettings projectServerSettings, LanguageDetection languageDetection, FilePreprocessor filePreprocessor, - ProjectExclusionFilters projectExclusionFilters, SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter) { + ProjectExclusionFilters projectExclusionFilters, SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter, HiddenFilesProjectData hiddenFilesProjectData) { this.analysisWarnings = analysisWarnings; this.scmConfiguration = scmConfiguration; this.inputModuleHierarchy = inputModuleHierarchy; @@ -92,6 +93,7 @@ public class ProjectFilePreprocessor { this.sonarGlobalPropertiesFilter = sonarGlobalPropertiesFilter; this.ignoreCommand = loadIgnoreCommand(); this.useScmExclusion = ignoreCommand != null; + this.hiddenFilesProjectData = hiddenFilesProjectData; } public void execute() { @@ -109,7 +111,7 @@ public class ProjectFilePreprocessor { int totalLanguagesDetected = languageDetection.getDetectedLanguages().size(); - progressReport.stop(String.format("%s detected in %s", pluralizeWithCount("language", totalLanguagesDetected), + progressReport.stopAndLogTotalTime(String.format("%s detected in %s", pluralizeWithCount("language", totalLanguagesDetected), pluralizeWithCount("preprocessed file", totalFilesPreprocessed))); int excludedFileByPatternCount = exclusionCounter.getByPatternsCount(); @@ -138,27 +140,31 @@ public class ProjectFilePreprocessor { // Default to index basedir when no sources provided List<Path> mainSourceDirsOrFiles = module.getSourceDirsOrFiles() .orElseGet(() -> hasChildModules || hasTests ? emptyList() : singletonList(module.getBaseDir().toAbsolutePath())); - List<Path> processedSources = processModuleSources(module, moduleExclusionFilters, mainSourceDirsOrFiles, InputFile.Type.MAIN, + List<Path> processedSources = processModuleSources(module, moduleConfig, moduleExclusionFilters, mainSourceDirsOrFiles, InputFile.Type.MAIN, exclusionCounter); mainSourcesByModule.put(module, processedSources); totalFilesPreprocessed += processedSources.size(); module.getTestDirsOrFiles().ifPresent(tests -> { - List<Path> processedTestSources = processModuleSources(module, moduleExclusionFilters, tests, InputFile.Type.TEST, exclusionCounter); + List<Path> processedTestSources = processModuleSources(module, moduleConfig, moduleExclusionFilters, tests, InputFile.Type.TEST, exclusionCounter); testSourcesByModule.put(module, processedTestSources); totalFilesPreprocessed += processedTestSources.size(); }); } - private List<Path> processModuleSources(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, List<Path> sources, + private List<Path> processModuleSources(DefaultInputModule module, ModuleConfiguration moduleConfiguration, ModuleExclusionFilters moduleExclusionFilters, List<Path> sources, InputFile.Type type, ExclusionCounter exclusionCounter) { List<Path> processedFiles = new ArrayList<>(); try { for (Path dirOrFile : sources) { if (dirOrFile.toFile().isDirectory()) { - processedFiles.addAll(processDirectory(module, moduleExclusionFilters, dirOrFile, type, exclusionCounter)); + processedFiles.addAll(processDirectory(module, moduleConfiguration, moduleExclusionFilters, dirOrFile, type, exclusionCounter)); } else { filePreprocessor.processFile(module, moduleExclusionFilters, dirOrFile, type, exclusionCounter, ignoreCommand) - .ifPresent(processedFiles::add); + .ifPresentOrElse( + processedFiles::add, + // If the file is not processed, we don't need to save visibility data and can remove it + () -> hiddenFilesProjectData.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(dirOrFile, module) + ); } } } catch (IOException e) { @@ -167,12 +173,17 @@ public class ProjectFilePreprocessor { return processedFiles; } - private List<Path> processDirectory(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, Path path, + private List<Path> processDirectory(DefaultInputModule module, ModuleConfiguration moduleConfiguration, ModuleExclusionFilters moduleExclusionFilters, Path path, InputFile.Type type, ExclusionCounter exclusionCounter) throws IOException { List<Path> processedFiles = new ArrayList<>(); Files.walkFileTree(path.normalize(), Collections.singleton(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, - new DirectoryFileVisitor(file -> filePreprocessor.processFile(module, moduleExclusionFilters, file, type, exclusionCounter, - ignoreCommand).ifPresent(processedFiles::add), module, moduleExclusionFilters, inputModuleHierarchy, type)); + new DirectoryFileVisitor(file -> filePreprocessor + .processFile(module, moduleExclusionFilters, file, type, exclusionCounter, ignoreCommand) + .ifPresentOrElse( + processedFiles::add, + // If the file is not processed, we don't need to save visibility data and can remove it + () -> hiddenFilesProjectData.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(file, module)), + module, moduleConfiguration, moduleExclusionFilters, inputModuleHierarchy, type, hiddenFilesProjectData)); return processedFiles; } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/AbstractSensorWrapper.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/AbstractSensorWrapper.java index a08380cf9d8..10d75a4b3c5 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/AbstractSensorWrapper.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/AbstractSensorWrapper.java @@ -19,11 +19,11 @@ */ package org.sonar.scanner.sensor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.sonar.api.batch.sensor.SensorContext; import org.sonar.api.batch.sensor.internal.DefaultSensorDescriptor; import org.sonar.api.scanner.sensor.ProjectSensor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.sonar.scanner.scan.branch.BranchConfiguration; import org.sonar.scanner.scan.branch.BranchType; import org.sonar.scanner.scan.filesystem.MutableFileSystem; @@ -60,7 +60,12 @@ public abstract class AbstractSensorWrapper<G extends ProjectSensor> { if (sensorIsRestricted) { LOGGER.info("Sensor {} is restricted to changed files only", descriptor.name()); } + boolean allowHiddenFileAnalysis = descriptor.isProcessesHiddenFiles(); + if (allowHiddenFileAnalysis) { + LOGGER.debug("Sensor {} is allowed to analyze hidden files", descriptor.name()); + } fileSystem.setRestrictToChangedFiles(sensorIsRestricted); + fileSystem.setAllowHiddenFileAnalysis(allowHiddenFileAnalysis); wrappedSensor.execute(context); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java index 5f28e7e283e..01b6c0c11cd 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java @@ -28,7 +28,6 @@ import org.sonar.api.batch.rule.ActiveRules; import org.sonar.api.batch.sensor.cache.ReadCache; import org.sonar.api.batch.sensor.cache.WriteCache; import org.sonar.api.config.Configuration; -import org.sonar.api.config.Settings; import org.sonar.scanner.bootstrap.ScannerPluginRepository; import org.sonar.scanner.cache.AnalysisCacheEnabled; import org.sonar.scanner.scan.branch.BranchConfiguration; @@ -38,11 +37,11 @@ public class ModuleSensorContext extends ProjectSensorContext { private final InputModule module; - public ModuleSensorContext(DefaultInputProject project, InputModule module, Configuration config, Settings mutableModuleSettings, FileSystem fs, ActiveRules activeRules, + public ModuleSensorContext(DefaultInputProject project, InputModule module, Configuration config, FileSystem fs, ActiveRules activeRules, DefaultSensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration, WriteCache writeCache, ReadCache readCache, AnalysisCacheEnabled analysisCacheEnabled, UnchangedFilesHandler unchangedFilesHandler, ExecutingSensorContext executingSensorContext, ScannerPluginRepository pluginRepository) { - super(project, config, mutableModuleSettings, fs, activeRules, sensorStorage, sonarRuntime, branchConfiguration, writeCache, readCache, analysisCacheEnabled, + super(project, config, fs, activeRules, sensorStorage, sonarRuntime, branchConfiguration, writeCache, readCache, analysisCacheEnabled, unchangedFilesHandler, executingSensorContext, pluginRepository); this.module = module; } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java index 6fb38fa4563..54c86750eaf 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java @@ -66,7 +66,6 @@ public class ProjectSensorContext implements SensorContext { static final NoOpNewAnalysisError NO_OP_NEW_ANALYSIS_ERROR = new NoOpNewAnalysisError(); - private final Settings mutableSettings; private final FileSystem fs; private final ActiveRules activeRules; private final DefaultSensorStorage sensorStorage; @@ -81,15 +80,14 @@ public class ProjectSensorContext implements SensorContext { private final ExecutingSensorContext executingSensorContext; private final ScannerPluginRepository pluginRepo; - public ProjectSensorContext(DefaultInputProject project, Configuration config, Settings mutableSettings, FileSystem fs, - ActiveRules activeRules, - DefaultSensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration, - WriteCache writeCache, ReadCache readCache, - AnalysisCacheEnabled analysisCacheEnabled, UnchangedFilesHandler unchangedFilesHandler, - ExecutingSensorContext executingSensorContext, ScannerPluginRepository pluginRepo) { + public ProjectSensorContext(DefaultInputProject project, Configuration config, FileSystem fs, + ActiveRules activeRules, + DefaultSensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration, + WriteCache writeCache, ReadCache readCache, + AnalysisCacheEnabled analysisCacheEnabled, UnchangedFilesHandler unchangedFilesHandler, + ExecutingSensorContext executingSensorContext, ScannerPluginRepository pluginRepo) { this.project = project; this.config = config; - this.mutableSettings = mutableSettings; this.fs = fs; this.activeRules = activeRules; this.sensorStorage = sensorStorage; @@ -105,7 +103,7 @@ public class ProjectSensorContext implements SensorContext { @Override public Settings settings() { - return mutableSettings; + throw new UnsupportedOperationException("This method is not supported anymore"); } @Override diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/CompositeBlameCommand.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/CompositeBlameCommand.java index 0742740bba6..a481f4a54f4 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/CompositeBlameCommand.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/CompositeBlameCommand.java @@ -22,7 +22,9 @@ package org.sonar.scm.git; import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.time.Instant; import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -32,6 +34,7 @@ import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.diff.RawTextComparator; @@ -238,7 +241,7 @@ public class CompositeBlameCommand extends BlameCommand { break; } linesList.add(new BlameLine() - .date(fileBlame.getCommitDates()[i]) + .date(toDate(fileBlame.getCommitDates()[i])) .revision(fileBlame.getCommitHashes()[i]) .author(fileBlame.getAuthorEmails()[i])); } @@ -251,4 +254,8 @@ public class CompositeBlameCommand extends BlameCommand { } } + private static @Nullable Date toDate(@Nullable Instant commitDate) { + return commitDate != null ? Date.from(commitDate) : null; + } + } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/NativeGitBlameCommand.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/NativeGitBlameCommand.java index 569999a55ab..8f066727e21 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/NativeGitBlameCommand.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/NativeGitBlameCommand.java @@ -25,6 +25,7 @@ import java.time.Instant; import java.util.Date; import java.util.LinkedList; import java.util.List; +import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -62,6 +63,7 @@ public class NativeGitBlameCommand { private final System2 system; private final ProcessWrapperFactory processWrapperFactory; + private final Consumer<String> stderrConsumer = line -> LOG.debug("[stderr] {}", line); private String gitCommand; @Autowired @@ -85,7 +87,7 @@ public class NativeGitBlameCommand { try { this.gitCommand = locateDefaultGit(); MutableString stdOut = new MutableString(); - this.processWrapperFactory.create(null, l -> stdOut.string = l, gitCommand, "--version").execute(); + this.processWrapperFactory.create(null, l -> stdOut.string = l, stderrConsumer, gitCommand, "--version").execute(); return stdOut.string != null && stdOut.string.startsWith("git version") && isCompatibleGitVersion(stdOut.string); } catch (Exception e) { LOG.debug("Failed to find git native client", e); @@ -109,7 +111,7 @@ public class NativeGitBlameCommand { // To avoid it we use where.exe to find git binary only in PATH. LOG.debug("Looking for git command in the PATH using where.exe (Windows)"); List<String> whereCommandResult = new LinkedList<>(); - this.processWrapperFactory.create(null, whereCommandResult::add, "C:\\Windows\\System32\\where.exe", "$PATH:git.exe") + this.processWrapperFactory.create(null, whereCommandResult::add, stderrConsumer, "C:\\Windows\\System32\\where.exe", "$PATH:git.exe") .execute(); if (!whereCommandResult.isEmpty()) { @@ -125,6 +127,7 @@ public class NativeGitBlameCommand { var processWrapper = this.processWrapperFactory.create( baseDir, outputProcessor::process, + stderrConsumer, gitCommand, GIT_DIR_FLAG, String.format(GIT_DIR_ARGUMENT, baseDir), GIT_DIR_FORCE_FLAG, baseDir.toString(), BLAME_COMMAND, |