From 4d04d1b26abaf409d0c68b97f0cac05e251ff16d Mon Sep 17 00:00:00 2001 From: Matteo Mara Date: Fri, 15 Dec 2023 14:31:39 +0100 Subject: SONAR-21195 Refactor file indexing into two distinct steps --- .../sonar/scanner/bootstrap/PluginInstaller.java | 11 +- .../scanner/bootstrap/ScannerPluginInstaller.java | 11 +- .../scanner/bootstrap/ScannerPluginRepository.java | 20 +- .../scanner/bootstrap/SpringScannerContainer.java | 12 +- .../scanner/mediumtest/FakePluginInstaller.java | 21 +- .../org/sonar/scanner/mediumtest/LocalPlugin.java | 33 +++ .../scanner/scan/SpringProjectScanContainer.java | 15 +- .../scan/filesystem/DirectoryFileVisitor.java | 152 ++++++++++++++ .../sonar/scanner/scan/filesystem/FileIndexer.java | 141 +++---------- .../scanner/scan/filesystem/FilePreprocessor.java | 139 ++++++++++++ .../scan/filesystem/InputFileFilterRepository.java | 34 +++ .../scanner/scan/filesystem/LanguageDetection.java | 16 +- .../scan/filesystem/ModuleRelativePathWarner.java | 47 +++++ .../scan/filesystem/ProjectFileIndexer.java | 232 +++------------------ .../scan/filesystem/ProjectFilePreprocessor.java | 230 ++++++++++++++++++++ 15 files changed, 771 insertions(+), 343 deletions(-) create mode 100644 sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/LocalPlugin.java create mode 100644 sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java create mode 100644 sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java create mode 100644 sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepository.java create mode 100644 sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleRelativePathWarner.java create mode 100644 sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java (limited to 'sonar-scanner-engine/src/main/java/org/sonar/scanner') diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java index eeb35462dab..de69726669c 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java @@ -22,6 +22,7 @@ package org.sonar.scanner.bootstrap; import java.util.List; import java.util.Map; import java.util.Set; +import org.sonar.scanner.mediumtest.LocalPlugin; public interface PluginInstaller { @@ -45,8 +46,14 @@ public interface PluginInstaller { Map installPluginsForLanguages(Set languageKeys); /** - * Used only by medium tests. + * Used only by medium tests. Installs required plugins (phase 1) * @see org.sonar.scanner.mediumtest.ScannerMediumTester */ - List installLocals(); + List installLocals(); + + /** + * Used only by medium tests. Installs optional plugins (phase 2) + * @see org.sonar.scanner.mediumtest.ScannerMediumTester + */ + List installOptionalLocals(Set languageKeys); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java index efed3403147..e05b7d08471 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java @@ -36,6 +36,7 @@ import org.sonar.api.utils.log.Loggers; import org.sonar.api.utils.log.Profiler; import org.sonar.core.platform.PluginInfo; import org.sonar.core.plugin.PluginType; +import org.sonar.scanner.mediumtest.LocalPlugin; import org.sonarqube.ws.client.GetRequest; import static java.lang.String.format; @@ -137,7 +138,15 @@ public class ScannerPluginInstaller implements PluginInstaller { * Returns empty on purpose. This method is used only by medium tests. */ @Override - public List installLocals() { + public List installLocals() { + return Collections.emptyList(); + } + + /** + * Returns empty on purpose. This method is used only by medium tests. + */ + @Override + public List installOptionalLocals(Set languageKeys) { return Collections.emptyList(); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java index 3ce0202292b..074079f974e 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java @@ -36,6 +36,7 @@ import org.sonar.core.platform.PluginInfo; import org.sonar.core.platform.PluginJarExploder; import org.sonar.core.platform.PluginRepository; import org.sonar.core.plugin.PluginType; +import org.sonar.scanner.mediumtest.LocalPlugin; import static java.util.stream.Collectors.toMap; import static org.sonar.api.utils.Preconditions.checkState; @@ -79,11 +80,11 @@ public class ScannerPluginRepository implements PluginRepository, Startable { pluginInstancesByKeys = new HashMap<>(loader.load(explodedPluginsByKey)); // this part is only used by medium tests - for (Object[] localPlugin : installer.installLocals()) { - String pluginKey = (String) localPlugin[0]; - PluginInfo pluginInfo = new PluginInfo(pluginKey); - pluginsByKeys.put(pluginKey, new ScannerPlugin(pluginInfo.getKey(), (long) localPlugin[2], PluginType.BUNDLED, pluginInfo)); - pluginInstancesByKeys.put(pluginKey, (Plugin) localPlugin[1]); + for (LocalPlugin localPlugin : installer.installLocals()) { + ScannerPlugin scannerPlugin = localPlugin.toScannerPlugin(); + String pluginKey = localPlugin.pluginKey(); + pluginsByKeys.put(pluginKey, scannerPlugin); + pluginInstancesByKeys.put(pluginKey, localPlugin.pluginInstance()); } keysByClassLoader = new HashMap<>(); @@ -107,6 +108,15 @@ public class ScannerPluginRepository implements PluginRepository, Startable { .collect(toMap(Map.Entry::getKey, e -> pluginJarExploder.explode(e.getValue().getInfo()))); pluginInstancesByKeys.putAll(new HashMap<>(loader.load(explodedPluginsByKey))); + // this part is only used by medium tests + for (LocalPlugin localPlugin : installer.installOptionalLocals(languageKeys)) { + ScannerPlugin scannerPlugin = localPlugin.toScannerPlugin(); + String pluginKey = localPlugin.pluginKey(); + languagePluginsByKeys.put(pluginKey, scannerPlugin); + pluginsByKeys.put(pluginKey, scannerPlugin); + pluginInstancesByKeys.put(pluginKey, localPlugin.pluginInstance()); + } + keysByClassLoader = new HashMap<>(); for (Map.Entry e : pluginInstancesByKeys.entrySet()) { keysByClassLoader.put(e.getValue().getClass().getClassLoader(), e.getKey()); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java index eedc07f9924..38972c95071 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java @@ -112,13 +112,14 @@ import org.sonar.scanner.scan.branch.BranchConfigurationProvider; import org.sonar.scanner.scan.branch.BranchType; import org.sonar.scanner.scan.branch.ProjectBranchesProvider; import org.sonar.scanner.scan.filesystem.DefaultProjectFileSystem; -import org.sonar.scanner.scan.filesystem.FileIndexer; +import org.sonar.scanner.scan.filesystem.FilePreprocessor; import org.sonar.scanner.scan.filesystem.InputComponentStore; import org.sonar.scanner.scan.filesystem.LanguageDetection; import org.sonar.scanner.scan.filesystem.MetadataGenerator; +import org.sonar.scanner.scan.filesystem.ModuleRelativePathWarner; import org.sonar.scanner.scan.filesystem.ProjectCoverageAndDuplicationExclusions; import org.sonar.scanner.scan.filesystem.ProjectExclusionFilters; -import org.sonar.scanner.scan.filesystem.ProjectFileIndexer; +import org.sonar.scanner.scan.filesystem.ProjectFilePreprocessor; import org.sonar.scanner.scan.filesystem.ScannerComponentIdGenerator; import org.sonar.scanner.scan.filesystem.StatusDetection; import org.sonar.scanner.scan.measure.DefaultMetricFinder; @@ -193,8 +194,9 @@ public class SpringScannerContainer extends SpringComponentContainer { LanguageDetection.class, MetadataGenerator.class, FileMetadata.class, - FileIndexer.class, - ProjectFileIndexer.class, + ModuleRelativePathWarner.class, + FilePreprocessor.class, + ProjectFilePreprocessor.class, ProjectExclusionFilters.class, // rules @@ -335,7 +337,7 @@ public class SpringScannerContainer extends SpringComponentContainer { getComponentByType(DeprecatedPropertiesWarningGenerator.class).execute(); - getComponentByType(ProjectFileIndexer.class).index(); + getComponentByType(ProjectFilePreprocessor.class).execute(); new SpringProjectScanContainer(this).execute(); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java index fb48e11b424..5aaabab3980 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java @@ -36,15 +36,21 @@ import org.sonar.scanner.bootstrap.ScannerPlugin; public class FakePluginInstaller implements PluginInstaller { private final Map pluginsByKeys = new HashMap<>(); - private final List mediumTestPlugins = new ArrayList<>(); + private final List mediumTestPlugins = new ArrayList<>(); + private final List optionalMediumTestPlugins = new ArrayList<>(); public FakePluginInstaller add(String pluginKey, File jarFile, long lastUpdatedAt) { pluginsByKeys.put(pluginKey, new ScannerPlugin(pluginKey, lastUpdatedAt, PluginType.BUNDLED, PluginInfo.create(jarFile))); return this; } - public FakePluginInstaller add(String pluginKey, Plugin instance, long lastUpdatedAt) { - mediumTestPlugins.add(new Object[] {pluginKey, instance, lastUpdatedAt}); + public FakePluginInstaller add(String pluginKey, Plugin instance) { + mediumTestPlugins.add(new LocalPlugin(pluginKey, instance, Set.of())); + return this; + } + + public FakePluginInstaller addOptional(String pluginKey, Set requiredForLanguages, Plugin instance) { + optionalMediumTestPlugins.add(new LocalPlugin(pluginKey, instance, requiredForLanguages)); return this; } @@ -64,7 +70,14 @@ public class FakePluginInstaller implements PluginInstaller { } @Override - public List installLocals() { + public List installLocals() { return mediumTestPlugins; } + + @Override + public List installOptionalLocals(Set languageKeys) { + return optionalMediumTestPlugins.stream() + .filter(plugin -> languageKeys.stream().anyMatch(lang -> plugin.requiredForLanguages().contains(lang))) + .toList(); + } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/LocalPlugin.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/LocalPlugin.java new file mode 100644 index 00000000000..20bd4895973 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/LocalPlugin.java @@ -0,0 +1,33 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.mediumtest; + +import java.util.Set; +import org.sonar.api.Plugin; +import org.sonar.core.platform.PluginInfo; +import org.sonar.core.plugin.PluginType; +import org.sonar.scanner.bootstrap.ScannerPlugin; + +public record LocalPlugin(String pluginKey, Plugin pluginInstance, Set requiredForLanguages) { + + public ScannerPlugin toScannerPlugin() { + return new ScannerPlugin(pluginKey, 1L, PluginType.BUNDLED, new PluginInfo(pluginKey)); + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java index 351593e9404..b70fde51660 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java @@ -49,7 +49,10 @@ import org.sonar.scanner.qualitygate.QualityGateCheck; import org.sonar.scanner.report.ContextPropertiesPublisher; import org.sonar.scanner.report.ReportPublisher; import org.sonar.scanner.rule.QProfileVerifier; -import org.sonar.scanner.scan.filesystem.InputComponentStore; +import org.sonar.scanner.scan.filesystem.FileIndexer; +import org.sonar.scanner.scan.filesystem.InputFileFilterRepository; +import org.sonar.scanner.scan.filesystem.LanguageDetection; +import org.sonar.scanner.scan.filesystem.ProjectFileIndexer; import org.sonar.scanner.scm.ScmPublisher; import org.sonar.scanner.sensor.ProjectSensorExtensionDictionary; import org.sonar.scanner.sensor.ProjectSensorsExecutor; @@ -69,7 +72,7 @@ public class SpringProjectScanContainer extends SpringComponentContainer { @Override protected void doBeforeStart() { - Set languages = getParentComponentByType(InputComponentStore.class).languages(); + Set languages = getParentComponentByType(LanguageDetection.class).getDetectedLanguages(); installPluginsForLanguages(languages); addScannerComponents(); } @@ -114,7 +117,12 @@ public class SpringProjectScanContainer extends SpringComponentContainer { ProjectSensorExtensionDictionary.class, ProjectSensorsExecutor.class, - AnalysisObservers.class); + AnalysisObservers.class, + + // file system + InputFileFilterRepository.class, + FileIndexer.class, + ProjectFileIndexer.class); } static ExtensionMatcher getScannerProjectExtensionsFilter() { @@ -130,6 +138,7 @@ public class SpringProjectScanContainer extends SpringComponentContainer { protected void doAfterStart() { getParentComponentByType(ScannerMetrics.class).addPluginMetrics(getComponentsByType(Metrics.class)); getComponentByType(ProjectLock.class).tryLock(); + getComponentByType(ProjectFileIndexer.class).index(); GlobalAnalysisMode analysisMode = getComponentByType(GlobalAnalysisMode.class); InputModuleHierarchy tree = getComponentByType(InputModuleHierarchy.class); ScanProperties properties = getComponentByType(ScanProperties.class); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java new file mode 100644 index 00000000000..eec0ed9c8d0 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java @@ -0,0 +1,152 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.scan.filesystem; + +import java.io.IOException; +import java.nio.file.AccessDeniedException; +import java.nio.file.FileSystemLoopException; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.DosFileAttributes; +import org.apache.commons.lang.SystemUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.scanner.fs.InputModuleHierarchy; + +public class DirectoryFileVisitor implements FileVisitor { + + private static final Logger LOG = LoggerFactory.getLogger(DirectoryFileVisitor.class); + + private final FileVisitAction fileVisitAction; + private final DefaultInputModule module; + private final ModuleExclusionFilters moduleExclusionFilters; + + private final InputModuleHierarchy inputModuleHierarchy; + private final InputFile.Type type; + + DirectoryFileVisitor(FileVisitAction fileVisitAction, DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, InputModuleHierarchy inputModuleHierarchy, InputFile.Type type) { + this.fileVisitAction = fileVisitAction; + this.module = module; + this.moduleExclusionFilters = moduleExclusionFilters; + this.inputModuleHierarchy = inputModuleHierarchy; + this.type = type; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + return isHidden(dir) ? FileVisitResult.SKIP_SUBTREE : FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (!Files.isHidden(file)) { + fileVisitAction.execute(file); + } + return FileVisitResult.CONTINUE; + } + + /** + *

Overridden method to handle exceptions while visiting files in the analysis.

+ * + *

+ *

    + *
  • FileSystemLoopException - We show a warning that a symlink loop exists and we skip the file.
    • + *
  • AccessDeniedException for excluded files/directories - We skip the file, as files excluded from the analysis, shouldn't throw access exceptions.
    • + *
+ *

+ * + * @param file a reference to the file + * @param exc the I/O exception that prevented the file from being visited + * @throws IOException + */ + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { + if (exc instanceof FileSystemLoopException) { + LOG.warn("Not indexing due to symlink loop: {}", file.toFile()); + return FileVisitResult.CONTINUE; + } else if (exc instanceof AccessDeniedException && isExcluded(file)) { + return FileVisitResult.CONTINUE; + } + throw exc; + } + + /** + *

Checks if the directory is excluded in the analysis or not. Only the exclusions are checked.

+ * + *

The inclusions cannot be checked for directories, since the current implementation of pattern matching is intended only for files.

+ * + * @param path The file or directory. + * @return True if file/directory is excluded from the analysis, false otherwise. + */ + private boolean isExcluded(Path path) throws IOException { + Path realAbsoluteFile = path.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize(); + return isExcludedDirectory(moduleExclusionFilters, realAbsoluteFile, inputModuleHierarchy.root().getBaseDir(), module.getBaseDir(), type); + } + + /** + *

Checks if the path is a directory that is excluded.

+ * + *

Exclusions patterns are checked both at project and module level.

+ * + * @param moduleExclusionFilters The exclusion filters. + * @param realAbsoluteFile The path to be checked. + * @param projectBaseDir The project base directory. + * @param moduleBaseDir The module base directory. + * @param type The input file type. + * @return True if path is an excluded directory, false otherwise. + */ + private static boolean isExcludedDirectory(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectBaseDir, Path moduleBaseDir, + InputFile.Type type) { + Path projectRelativePath = projectBaseDir.relativize(realAbsoluteFile); + Path moduleRelativePath = moduleBaseDir.relativize(realAbsoluteFile); + return moduleExclusionFilters.isExcludedAsParentDirectoryOfExcludedChildren(realAbsoluteFile, projectRelativePath, projectBaseDir, type) + || moduleExclusionFilters.isExcludedAsParentDirectoryOfExcludedChildren(realAbsoluteFile, moduleRelativePath, moduleBaseDir, type); + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) { + return FileVisitResult.CONTINUE; + } + + private static boolean isHidden(Path path) throws IOException { + if (SystemUtils.IS_OS_WINDOWS) { + try { + DosFileAttributes dosFileAttributes = Files.readAttributes(path, DosFileAttributes.class, LinkOption.NOFOLLOW_LINKS); + return dosFileAttributes.isHidden(); + } catch (UnsupportedOperationException e) { + return path.toFile().isHidden(); + } + } else { + return Files.isHidden(path); + } + } + + @FunctionalInterface + interface FileVisitAction { + void execute(Path file) throws IOException; + } +} + diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java index 4d9b8c2a377..4d264590716 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java @@ -19,14 +19,10 @@ */ package org.sonar.scanner.scan.filesystem; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.LinkOption; import java.nio.file.Path; import java.util.Arrays; -import java.util.function.BooleanSupplier; -import javax.annotation.Nullable; -import org.apache.commons.io.FilenameUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.sonar.api.CoreProperties; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.InputFile.Type; @@ -36,11 +32,7 @@ import org.sonar.api.batch.fs.internal.DefaultInputFile; import org.sonar.api.batch.fs.internal.DefaultInputModule; import org.sonar.api.batch.fs.internal.DefaultInputProject; import org.sonar.api.batch.fs.internal.SensorStrategy; -import org.sonar.api.batch.scm.IgnoreCommand; -import org.sonar.api.notifications.AnalysisWarnings; import org.sonar.api.utils.MessageException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.sonar.scanner.issue.ignore.scanner.IssueExclusionsLoader; import org.sonar.scanner.repository.language.Language; import org.sonar.scanner.scan.ScanProperties; @@ -56,10 +48,7 @@ public class FileIndexer { private static final Logger LOG = LoggerFactory.getLogger(FileIndexer.class); - private final AnalysisWarnings analysisWarnings; private final ScanProperties properties; - private final InputFileFilter[] filters; - private final ProjectExclusionFilters projectExclusionFilters; private final ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions; private final IssueExclusionsLoader issueExclusionsLoader; private final MetadataGenerator metadataGenerator; @@ -71,15 +60,13 @@ public class FileIndexer { private final StatusDetection statusDetection; private final ScmChangedFiles scmChangedFiles; - private boolean warnInclusionsAlreadyLogged; - private boolean warnExclusionsAlreadyLogged; - private boolean warnCoverageExclusionsAlreadyLogged; - private boolean warnDuplicationExclusionsAlreadyLogged; + private final ModuleRelativePathWarner moduleRelativePathWarner; + private final InputFileFilterRepository inputFileFilterRepository; public FileIndexer(DefaultInputProject project, ScannerComponentIdGenerator scannerComponentIdGenerator, InputComponentStore componentStore, - ProjectExclusionFilters projectExclusionFilters, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, IssueExclusionsLoader issueExclusionsLoader, - MetadataGenerator metadataGenerator, SensorStrategy sensorStrategy, LanguageDetection languageDetection, AnalysisWarnings analysisWarnings, ScanProperties properties, - InputFileFilter[] filters, ScmChangedFiles scmChangedFiles, StatusDetection statusDetection) { + ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, IssueExclusionsLoader issueExclusionsLoader, + MetadataGenerator metadataGenerator, SensorStrategy sensorStrategy, LanguageDetection languageDetection, ScanProperties properties, + ScmChangedFiles scmChangedFiles, StatusDetection statusDetection, ModuleRelativePathWarner moduleRelativePathWarner, InputFileFilterRepository inputFileFilterRepository) { this.project = project; this.scannerComponentIdGenerator = scannerComponentIdGenerator; this.componentStore = componentStore; @@ -88,55 +75,23 @@ public class FileIndexer { this.metadataGenerator = metadataGenerator; this.sensorStrategy = sensorStrategy; this.langDetection = languageDetection; - this.analysisWarnings = analysisWarnings; this.properties = properties; - this.filters = filters; - this.projectExclusionFilters = projectExclusionFilters; this.scmChangedFiles = scmChangedFiles; this.statusDetection = statusDetection; + this.moduleRelativePathWarner = moduleRelativePathWarner; + this.inputFileFilterRepository = inputFileFilterRepository; } - void indexFile(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, - Path sourceFile, Type type, ProgressReport progressReport, ProjectFileIndexer.ExclusionCounter exclusionCounter, @Nullable IgnoreCommand ignoreCommand) - throws IOException { - // get case of real file without resolving link - Path realAbsoluteFile = sourceFile.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize(); - Path projectRelativePath = project.getBaseDir().relativize(realAbsoluteFile); - Path moduleRelativePath = module.getBaseDir().relativize(realAbsoluteFile); - boolean included = evaluateInclusionsFilters(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type); - if (!included) { - exclusionCounter.increaseByPatternsCount(); - return; - } - boolean excluded = evaluateExclusionsFilters(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type); - if (excluded) { - exclusionCounter.increaseByPatternsCount(); - return; - } - if (!realAbsoluteFile.startsWith(project.getBaseDir())) { - LOG.warn("File '{}' is ignored. It is not located in project basedir '{}'.", realAbsoluteFile.toAbsolutePath(), project.getBaseDir()); - return; - } - if (!realAbsoluteFile.startsWith(module.getBaseDir())) { - LOG.warn("File '{}' is ignored. It is not located in module basedir '{}'.", realAbsoluteFile.toAbsolutePath(), module.getBaseDir()); - return; - } - - if (Files.exists(realAbsoluteFile) && isFileSizeBiggerThanLimit(realAbsoluteFile)) { - LOG.warn("File '{}' is bigger than {}MB and as consequence is removed from the analysis scope.", realAbsoluteFile.toAbsolutePath(), properties.fileSizeLimit()); - return; - } - - Language language = langDetection.language(realAbsoluteFile, projectRelativePath); + void indexFile(DefaultInputModule module, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path sourceFile, + Type type, ProgressReport progressReport) { + Path projectRelativePath = project.getBaseDir().relativize(sourceFile); + Path moduleRelativePath = module.getBaseDir().relativize(sourceFile); - if (ignoreCommand != null && ignoreCommand.isIgnored(realAbsoluteFile)) { - LOG.debug("File '{}' is excluded by the scm ignore settings.", realAbsoluteFile); - exclusionCounter.increaseByScmCount(); - return; - } + // This should be fast; language should be cached from preprocessing step + Language language = langDetection.language(sourceFile, projectRelativePath); DefaultIndexedFile indexedFile = new DefaultIndexedFile( - realAbsoluteFile, + sourceFile, project.key(), projectRelativePath.toString(), moduleRelativePath.toString(), @@ -144,7 +99,7 @@ public class FileIndexer { language != null ? language.key() : null, scannerComponentIdGenerator.getAsInt(), sensorStrategy, - scmChangedFiles.getOldRelativeFilePath(realAbsoluteFile) + scmChangedFiles.getOldRelativeFilePath(sourceFile) ); DefaultInputFile inputFile = new DefaultInputFile(indexedFile, f -> metadataGenerator.setMetadata(module.key(), f, module.getEncoding()), @@ -159,7 +114,9 @@ public class FileIndexer { componentStore.put(module.key(), inputFile); issueExclusionsLoader.addMulticriteriaPatterns(inputFile); String langStr = inputFile.language() != null ? format("with language '%s'", inputFile.language()) : "with no language"; - LOG.debug("'{}' indexed {}{}", projectRelativePath, type == Type.TEST ? "as test " : "", langStr); + if (LOG.isDebugEnabled()) { + LOG.debug("'{}' indexed {}{}", projectRelativePath, type == Type.TEST ? "as test " : "", langStr); + } evaluateCoverageExclusions(moduleCoverageAndDuplicationExclusions, inputFile); evaluateDuplicationExclusions(moduleCoverageAndDuplicationExclusions, inputFile); if (properties.preloadFileMetadata()) { @@ -169,42 +126,6 @@ public class FileIndexer { progressReport.message(count + " " + pluralizeFiles(count) + " indexed... (last one was " + inputFile.getProjectRelativePath() + ")"); } - private boolean evaluateInclusionsFilters(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectRelativePath, Path moduleRelativePath, - InputFile.Type type) { - if (!Arrays.equals(moduleExclusionFilters.getInclusionsConfig(type), projectExclusionFilters.getInclusionsConfig(type))) { - // Module specific configuration - return moduleExclusionFilters.isIncluded(realAbsoluteFile, moduleRelativePath, type); - } - boolean includedByProjectConfiguration = projectExclusionFilters.isIncluded(realAbsoluteFile, projectRelativePath, type); - if (includedByProjectConfiguration) { - return true; - } else if (moduleExclusionFilters.isIncluded(realAbsoluteFile, moduleRelativePath, type)) { - warnOnce( - type == Type.MAIN ? CoreProperties.PROJECT_INCLUSIONS_PROPERTY : CoreProperties.PROJECT_TEST_INCLUSIONS_PROPERTY, - FilenameUtils.normalize(projectRelativePath.toString(), true), () -> warnInclusionsAlreadyLogged, () -> warnInclusionsAlreadyLogged = true); - return true; - } - return false; - } - - private boolean evaluateExclusionsFilters(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectRelativePath, Path moduleRelativePath, - InputFile.Type type) { - if (!Arrays.equals(moduleExclusionFilters.getExclusionsConfig(type), projectExclusionFilters.getExclusionsConfig(type))) { - // Module specific configuration - return moduleExclusionFilters.isExcluded(realAbsoluteFile, moduleRelativePath, type); - } - boolean includedByProjectConfiguration = projectExclusionFilters.isExcluded(realAbsoluteFile, projectRelativePath, type); - if (includedByProjectConfiguration) { - return true; - } else if (moduleExclusionFilters.isExcluded(realAbsoluteFile, moduleRelativePath, type)) { - warnOnce( - type == Type.MAIN ? CoreProperties.PROJECT_EXCLUSIONS_PROPERTY : CoreProperties.PROJECT_TEST_EXCLUSIONS_PROPERTY, - FilenameUtils.normalize(projectRelativePath.toString(), true), () -> warnExclusionsAlreadyLogged, () -> warnExclusionsAlreadyLogged = true); - return true; - } - return false; - } - private void checkIfAlreadyIndexed(DefaultInputFile inputFile) { if (componentStore.inputFile(inputFile.getProjectRelativePath()) != null) { throw MessageException.of("File " + inputFile + " can't be indexed twice. Please check that inclusion/exclusion patterns produce " @@ -229,8 +150,7 @@ public class FileIndexer { if (excludedByProjectConfiguration) { return true; } else if (moduleCoverageAndDuplicationExclusions.isExcludedForCoverage(inputFile)) { - warnOnce(CoreProperties.PROJECT_COVERAGE_EXCLUSIONS_PROPERTY, inputFile.getProjectRelativePath(), () -> warnCoverageExclusionsAlreadyLogged, - () -> warnCoverageExclusionsAlreadyLogged = true); + moduleRelativePathWarner.warnOnce(CoreProperties.PROJECT_COVERAGE_EXCLUSIONS_PROPERTY, inputFile.getProjectRelativePath()); return true; } return false; @@ -253,26 +173,15 @@ public class FileIndexer { if (excludedByProjectConfiguration) { return true; } else if (moduleCoverageAndDuplicationExclusions.isExcludedForDuplication(inputFile)) { - warnOnce(CoreProperties.CPD_EXCLUSIONS, inputFile.getProjectRelativePath(), () -> warnDuplicationExclusionsAlreadyLogged, - () -> warnDuplicationExclusionsAlreadyLogged = true); + moduleRelativePathWarner.warnOnce(CoreProperties.CPD_EXCLUSIONS, inputFile.getProjectRelativePath()); return true; } return false; } - private void warnOnce(String propKey, String filePath, BooleanSupplier alreadyLoggedGetter, Runnable markAsLogged) { - if (!alreadyLoggedGetter.getAsBoolean()) { - String msg = "Specifying module-relative paths at project level in the property '" + propKey + "' is deprecated. " + - "To continue matching files like '" + filePath + "', update this property so that patterns refer to project-relative paths."; - LOG.warn(msg); - analysisWarnings.addUnique(msg); - markAsLogged.run(); - } - } - private boolean accept(InputFile indexedFile) { // InputFileFilter extensions. Might trigger generation of metadata - for (InputFileFilter filter : filters) { + for (InputFileFilter filter : inputFileFilterRepository.getInputFileFilters()) { if (!filter.accept(indexedFile)) { LOG.debug("'{}' excluded by {}", indexedFile, filter.getClass().getName()); return false; @@ -285,7 +194,5 @@ public class FileIndexer { return count == 1 ? "file" : "files"; } - private boolean isFileSizeBiggerThanLimit(Path filePath) throws IOException { - return Files.size(filePath) > properties.fileSizeLimit() * 1024L * 1024L; - } + } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java new file mode 100644 index 00000000000..6faa87cc42a --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java @@ -0,0 +1,139 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.scan.filesystem; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Optional; +import javax.annotation.CheckForNull; +import org.apache.commons.io.FilenameUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.CoreProperties; +import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.api.batch.fs.internal.DefaultInputProject; +import org.sonar.api.batch.scm.IgnoreCommand; +import org.sonar.scanner.scan.ScanProperties; + +public class FilePreprocessor { + + private static final Logger LOG = LoggerFactory.getLogger(FilePreprocessor.class); + + private final ModuleRelativePathWarner moduleRelativePathWarner; + private final DefaultInputProject project; + private final LanguageDetection languageDetection; + private final ProjectExclusionFilters projectExclusionFilters; + private final ScanProperties properties; + + public FilePreprocessor(ModuleRelativePathWarner moduleRelativePathWarner, DefaultInputProject project, + LanguageDetection languageDetection, ProjectExclusionFilters projectExclusionFilters, ScanProperties properties) { + this.moduleRelativePathWarner = moduleRelativePathWarner; + this.project = project; + this.languageDetection = languageDetection; + this.projectExclusionFilters = projectExclusionFilters; + this.properties = properties; + } + + public Optional processFile(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, Path sourceFile, + InputFile.Type type, ProjectFilePreprocessor.ExclusionCounter exclusionCounter, @CheckForNull IgnoreCommand ignoreCommand) throws IOException { + // get case of real file without resolving link + Path realAbsoluteFile = sourceFile.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize(); + Path projectRelativePath = project.getBaseDir().relativize(realAbsoluteFile); + Path moduleRelativePath = module.getBaseDir().relativize(realAbsoluteFile); + boolean included = isFileIncluded(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type); + if (!included) { + exclusionCounter.increaseByPatternsCount(); + return Optional.empty(); + } + boolean excluded = isFileExcluded(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type); + if (excluded) { + exclusionCounter.increaseByPatternsCount(); + return Optional.empty(); + } + + if (!realAbsoluteFile.startsWith(project.getBaseDir())) { + LOG.warn("File '{}' is ignored. It is not located in project basedir '{}'.", realAbsoluteFile.toAbsolutePath(), project.getBaseDir()); + return Optional.empty(); + } + if (!realAbsoluteFile.startsWith(module.getBaseDir())) { + LOG.warn("File '{}' is ignored. It is not located in module basedir '{}'.", realAbsoluteFile.toAbsolutePath(), module.getBaseDir()); + return Optional.empty(); + } + + if (ignoreCommand != null && ignoreCommand.isIgnored(realAbsoluteFile)) { + LOG.debug("File '{}' is excluded by the scm ignore settings.", realAbsoluteFile); + exclusionCounter.increaseByScmCount(); + return Optional.empty(); + } + + if (Files.exists(realAbsoluteFile) && isFileSizeBiggerThanLimit(realAbsoluteFile)) { + LOG.warn("File '{}' is bigger than {}MB and as consequence is removed from the analysis scope.", realAbsoluteFile.toAbsolutePath(), properties.fileSizeLimit()); + return Optional.empty(); + } + + languageDetection.language(realAbsoluteFile, projectRelativePath); + + return Optional.of(realAbsoluteFile); + } + + private boolean isFileIncluded(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectRelativePath, + Path moduleRelativePath, InputFile.Type type) { + if (!Arrays.equals(moduleExclusionFilters.getInclusionsConfig(type), projectExclusionFilters.getInclusionsConfig(type))) { + return moduleExclusionFilters.isIncluded(realAbsoluteFile, moduleRelativePath, type); + } + boolean includedByProjectConfiguration = projectExclusionFilters.isIncluded(realAbsoluteFile, projectRelativePath, type); + if (includedByProjectConfiguration) { + return true; + } + if (moduleExclusionFilters.isIncluded(realAbsoluteFile, moduleRelativePath, type)) { + moduleRelativePathWarner.warnOnce( + type == InputFile.Type.MAIN ? CoreProperties.PROJECT_INCLUSIONS_PROPERTY : CoreProperties.PROJECT_TEST_INCLUSIONS_PROPERTY, + FilenameUtils.normalize(projectRelativePath.toString(), true)); + return true; + } + return false; + } + + private boolean isFileExcluded(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectRelativePath, + Path moduleRelativePath, InputFile.Type type) { + if (!Arrays.equals(moduleExclusionFilters.getExclusionsConfig(type), projectExclusionFilters.getExclusionsConfig(type))) { + return moduleExclusionFilters.isExcluded(realAbsoluteFile, moduleRelativePath, type); + } + boolean includedByProjectConfiguration = projectExclusionFilters.isExcluded(realAbsoluteFile, projectRelativePath, type); + if (includedByProjectConfiguration) { + return true; + } + if (moduleExclusionFilters.isExcluded(realAbsoluteFile, moduleRelativePath, type)) { + moduleRelativePathWarner.warnOnce( + type == InputFile.Type.MAIN ? CoreProperties.PROJECT_EXCLUSIONS_PROPERTY : CoreProperties.PROJECT_TEST_EXCLUSIONS_PROPERTY, + FilenameUtils.normalize(projectRelativePath.toString(), true)); + return true; + } + return false; + } + + private boolean isFileSizeBiggerThanLimit(Path filePath) throws IOException { + return Files.size(filePath) > properties.fileSizeLimit() * 1024L * 1024L; + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepository.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepository.java new file mode 100644 index 00000000000..b6e14af85eb --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepository.java @@ -0,0 +1,34 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.scan.filesystem; + +import org.sonar.api.batch.fs.InputFileFilter; + +public class InputFileFilterRepository { + private final InputFileFilter[] inputFileFilters; + + public InputFileFilterRepository(InputFileFilter... inputFileFilters) { + this.inputFileFilters = inputFileFilters; + } + + public InputFileFilter[] getInputFileFilters() { + return inputFileFilters; + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java index 5ede0250c08..24128133824 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java @@ -25,6 +25,7 @@ import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.CheckForNull; @@ -53,8 +54,9 @@ public class LanguageDetection { */ private final Map patternsByLanguage; private final List languagesToConsider; + private final Map languageCacheByPath; - public LanguageDetection(Configuration settings, LanguagesRepository languages) { + public LanguageDetection(Configuration settings, LanguagesRepository languages, Map languageCache) { Map patternsByLanguageBuilder = new LinkedHashMap<>(); for (Language language : languages.all()) { String[] filePatterns = settings.getStringArray(getFileLangPatternPropKey(language.key())); @@ -69,6 +71,7 @@ public class LanguageDetection { languagesToConsider = List.copyOf(patternsByLanguageBuilder.keySet()); patternsByLanguage = unmodifiableMap(patternsByLanguageBuilder); + languageCacheByPath = languageCache; } private static PathPattern[] getLanguagePatterns(Language language) { @@ -89,11 +92,16 @@ public class LanguageDetection { @CheckForNull Language language(Path absolutePath, Path relativePath) { - Language detectedLanguage = null; + Language detectedLanguage = languageCacheByPath.get(absolutePath.toString()); + if (detectedLanguage != null) { + return detectedLanguage; + } + for (Language language : languagesToConsider) { if (isCandidateForLanguage(absolutePath, relativePath, language)) { if (detectedLanguage == null) { detectedLanguage = language; + languageCacheByPath.put(absolutePath.toString(), language); } else { // Language was already forced by another pattern throw MessageException.of(MessageFormat.format("Language of file ''{0}'' can not be decided as the file matches patterns of both {1} and {2}", @@ -105,6 +113,10 @@ public class LanguageDetection { return detectedLanguage; } + public Set getDetectedLanguages() { + return languageCacheByPath.values().stream().map(Language::key).collect(Collectors.toSet()); + } + private boolean isCandidateForLanguage(Path absolutePath, Path relativePath, Language language) { PathPattern[] patterns = patternsByLanguage.get(language); return patterns != null && Arrays.stream(patterns).anyMatch(pattern -> pattern.match(absolutePath, relativePath, false)); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleRelativePathWarner.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleRelativePathWarner.java new file mode 100644 index 00000000000..766955e438c --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleRelativePathWarner.java @@ -0,0 +1,47 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.scan.filesystem; + +import java.util.HashSet; +import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.notifications.AnalysisWarnings; + +public class ModuleRelativePathWarner { + + private static final Logger LOG = LoggerFactory.getLogger(ModuleRelativePathWarner.class); + private final AnalysisWarnings analysisWarnings; + private final Set previouslyWarnedProps = new HashSet<>(); + + public ModuleRelativePathWarner(AnalysisWarnings analysisWarnings) { + this.analysisWarnings = analysisWarnings; + } + + public void warnOnce(String propKey, String filePath) { + if (!previouslyWarnedProps.contains(propKey)) { + previouslyWarnedProps.add(propKey); + String msg = "Specifying module-relative paths at project level in the property '" + propKey + "' is deprecated. " + + "To continue matching files like '" + filePath + "', update this property so that patterns refer to project-relative paths."; + LOG.warn(msg); + analysisWarnings.addUnique(msg); + } + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java index 35d5bf3fdb4..5ff7ca1ef17 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java @@ -20,33 +20,22 @@ package org.sonar.scanner.scan.filesystem; import java.io.IOException; -import java.nio.file.AccessDeniedException; -import java.nio.file.FileSystemLoopException; import java.nio.file.FileVisitOption; -import java.nio.file.FileVisitResult; -import java.nio.file.FileVisitor; import java.nio.file.Files; -import java.nio.file.LinkOption; import java.nio.file.Path; -import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.attribute.DosFileAttributes; import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.SystemUtils; -import org.sonar.api.batch.fs.InputFile; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.sonar.api.batch.fs.InputFile.Type; import org.sonar.api.batch.fs.internal.DefaultInputModule; -import org.sonar.api.batch.scm.IgnoreCommand; import org.sonar.api.notifications.AnalysisWarnings; import org.sonar.api.scan.filesystem.PathResolver; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.sonar.scanner.bootstrap.GlobalConfiguration; import org.sonar.scanner.bootstrap.GlobalServerSettings; import org.sonar.scanner.fs.InputModuleHierarchy; @@ -54,12 +43,8 @@ import org.sonar.scanner.scan.ModuleConfiguration; import org.sonar.scanner.scan.ModuleConfigurationProvider; import org.sonar.scanner.scan.ProjectServerSettings; import org.sonar.scanner.scan.SonarGlobalPropertiesFilter; -import org.sonar.scanner.scm.ScmConfiguration; import org.sonar.scanner.util.ProgressReport; -import static java.util.Collections.emptyList; -import static java.util.Collections.singletonList; - /** * Index project input files into {@link InputComponentStore}. */ @@ -69,15 +54,13 @@ public class ProjectFileIndexer { private final ProjectExclusionFilters projectExclusionFilters; private final SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter; private final ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions; - private final ScmConfiguration scmConfiguration; private final InputComponentStore componentStore; private final InputModuleHierarchy inputModuleHierarchy; private final GlobalConfiguration globalConfig; private final GlobalServerSettings globalServerSettings; private final ProjectServerSettings projectServerSettings; private final FileIndexer fileIndexer; - private final IgnoreCommand ignoreCommand; - private final boolean useScmExclusion; + private final ProjectFilePreprocessor projectFilePreprocessor; private final AnalysisWarnings analysisWarnings; private ProgressReport progressReport; @@ -85,8 +68,8 @@ public class ProjectFileIndexer { public ProjectFileIndexer(InputComponentStore componentStore, ProjectExclusionFilters exclusionFilters, SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter, InputModuleHierarchy inputModuleHierarchy, GlobalConfiguration globalConfig, GlobalServerSettings globalServerSettings, ProjectServerSettings projectServerSettings, - FileIndexer fileIndexer, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, ScmConfiguration scmConfiguration, - AnalysisWarnings analysisWarnings) { + FileIndexer fileIndexer, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, + ProjectFilePreprocessor projectFilePreprocessor, AnalysisWarnings analysisWarnings) { this.componentStore = componentStore; this.sonarGlobalPropertiesFilter = sonarGlobalPropertiesFilter; this.inputModuleHierarchy = inputModuleHierarchy; @@ -96,10 +79,8 @@ public class ProjectFileIndexer { this.fileIndexer = fileIndexer; this.projectExclusionFilters = exclusionFilters; this.projectCoverageAndDuplicationExclusions = projectCoverageAndDuplicationExclusions; - this.scmConfiguration = scmConfiguration; + this.projectFilePreprocessor = projectFilePreprocessor; this.analysisWarnings = analysisWarnings; - this.ignoreCommand = loadIgnoreCommand(); - this.useScmExclusion = ignoreCommand != null; } public void index() { @@ -108,47 +89,22 @@ public class ProjectFileIndexer { LOG.info("Project configuration:"); projectExclusionFilters.log(" "); projectCoverageAndDuplicationExclusions.log(" "); - ExclusionCounter exclusionCounter = new ExclusionCounter(); - if (useScmExclusion) { - ignoreCommand.init(inputModuleHierarchy.root().getBaseDir().toAbsolutePath()); - indexModulesRecursively(inputModuleHierarchy.root(), exclusionCounter); - ignoreCommand.clean(); - } else { - indexModulesRecursively(inputModuleHierarchy.root(), exclusionCounter); - } + indexModulesRecursively(inputModuleHierarchy.root()); int totalIndexed = componentStore.inputFiles().size(); progressReport.stop(totalIndexed + " " + pluralizeFiles(totalIndexed) + " indexed"); - int excludedFileByPatternCount = exclusionCounter.getByPatternsCount(); - if (projectExclusionFilters.hasPattern() || excludedFileByPatternCount > 0) { - LOG.info("{} {} ignored because of inclusion/exclusion patterns", excludedFileByPatternCount, pluralizeFiles(excludedFileByPatternCount)); - } - int excludedFileByScmCount = exclusionCounter.getByScmCount(); - if (useScmExclusion) { - LOG.info("{} {} ignored because of scm ignore settings", excludedFileByScmCount, pluralizeFiles(excludedFileByScmCount)); - } } - private IgnoreCommand loadIgnoreCommand() { - try { - if (!scmConfiguration.isExclusionDisabled() && scmConfiguration.provider() != null) { - return scmConfiguration.provider().ignoreCommand(); - } - } catch (UnsupportedOperationException e) { - LOG.debug("File exclusion based on SCM ignore information is not available with this plugin."); - } - - return null; + private void indexModulesRecursively(DefaultInputModule module) { + inputModuleHierarchy.children(module).stream() + .sorted(Comparator.comparing(DefaultInputModule::key)) + .forEach(this::indexModulesRecursively); + index(module); } - private void indexModulesRecursively(DefaultInputModule module, ExclusionCounter exclusionCounter) { - inputModuleHierarchy.children(module).stream().sorted(Comparator.comparing(DefaultInputModule::key)).forEach(m -> indexModulesRecursively(m, exclusionCounter)); - index(module, exclusionCounter); - } - - private void index(DefaultInputModule module, ExclusionCounter exclusionCounter) { + private void index(DefaultInputModule module) { // Emulate creation of module level settings ModuleConfiguration moduleConfig = new ModuleConfigurationProvider(sonarGlobalPropertiesFilter).provide(globalConfig, module, globalServerSettings, projectServerSettings); ModuleExclusionFilters moduleExclusionFilters = new ModuleExclusionFilters(moduleConfig, analysisWarnings); @@ -161,13 +117,10 @@ public class ProjectFileIndexer { moduleExclusionFilters.log(" "); moduleCoverageAndDuplicationExclusions.log(" "); } - boolean hasChildModules = !module.definition().getSubProjects().isEmpty(); - boolean hasTests = module.getTestDirsOrFiles().isPresent(); - // Default to index basedir when no sources provided - List mainSourceDirsOrFiles = module.getSourceDirsOrFiles() - .orElseGet(() -> hasChildModules || hasTests ? emptyList() : singletonList(module.getBaseDir().toAbsolutePath())); - indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, mainSourceDirsOrFiles, Type.MAIN, exclusionCounter); - module.getTestDirsOrFiles().ifPresent(tests -> indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, tests, Type.TEST, exclusionCounter)); + List mainSourceDirsOrFiles = projectFilePreprocessor.getMainSourcesByModule(module); + indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, mainSourceDirsOrFiles, Type.MAIN); + projectFilePreprocessor.getTestSourcesByModule(module) + .ifPresent(tests -> indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, tests, Type.TEST)); } private static void logPaths(String label, Path baseDir, List paths) { @@ -176,7 +129,7 @@ public class ProjectFileIndexer { for (Iterator it = paths.iterator(); it.hasNext(); ) { Path file = it.next(); Optional relativePathToBaseDir = PathResolver.relativize(baseDir, file); - if (!relativePathToBaseDir.isPresent()) { + if (relativePathToBaseDir.isEmpty()) { sb.append(file); } else if (StringUtils.isBlank(relativePathToBaseDir.get())) { sb.append("."); @@ -195,19 +148,14 @@ public class ProjectFileIndexer { } } - private static String pluralizeFiles(int count) { - return count == 1 ? "file" : "files"; - } - - private void indexFiles(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, - ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, List sources, Type type, ExclusionCounter exclusionCounter) { + private void indexFiles(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, + List sources, Type type) { try { for (Path dirOrFile : sources) { if (dirOrFile.toFile().isDirectory()) { - indexDirectory(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type, exclusionCounter); + indexDirectory(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type); } else { - fileIndexer.indexFile(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type, progressReport, exclusionCounter, - ignoreCommand); + fileIndexer.indexFile(module, moduleCoverageAndDuplicationExclusions, dirOrFile, type, progressReport); } } } catch (IOException e) { @@ -216,141 +164,17 @@ public class ProjectFileIndexer { } private void indexDirectory(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, - ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path dirToIndex, Type type, ExclusionCounter exclusionCounter) - throws IOException { + ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, + Path dirToIndex, Type type) throws IOException { Files.walkFileTree(dirToIndex.normalize(), Collections.singleton(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, - new IndexFileVisitor(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, type, exclusionCounter)); - } - - - /** - *

Checks if the path is a directory that is excluded.

- * - *

Exclusions patterns are checked both at project and module level.

- * - * @param moduleExclusionFilters The exclusion filters. - * @param realAbsoluteFile The path to be checked. - * @param projectBaseDir The project base directory. - * @param moduleBaseDir The module base directory. - * @param type The input file type. - * @return True if path is an excluded directory, false otherwise. - */ - private static boolean isExcludedDirectory(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectBaseDir, Path moduleBaseDir, - InputFile.Type type) { - Path projectRelativePath = projectBaseDir.relativize(realAbsoluteFile); - Path moduleRelativePath = moduleBaseDir.relativize(realAbsoluteFile); - return moduleExclusionFilters.isExcludedAsParentDirectoryOfExcludedChildren(realAbsoluteFile, projectRelativePath, projectBaseDir, type) - || moduleExclusionFilters.isExcludedAsParentDirectoryOfExcludedChildren(realAbsoluteFile, moduleRelativePath, moduleBaseDir, type); + new DirectoryFileVisitor(file -> fileIndexer.indexFile(module, moduleCoverageAndDuplicationExclusions, file, type, progressReport), + module, moduleExclusionFilters, inputModuleHierarchy, type)); } - private class IndexFileVisitor implements FileVisitor { - private final DefaultInputModule module; - private final ModuleExclusionFilters moduleExclusionFilters; - private final ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions; - private final Type type; - private final ExclusionCounter exclusionCounter; - - IndexFileVisitor(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, - Type type, - ExclusionCounter exclusionCounter) { - this.module = module; - this.moduleExclusionFilters = moduleExclusionFilters; - this.moduleCoverageAndDuplicationExclusions = moduleCoverageAndDuplicationExclusions; - this.type = type; - this.exclusionCounter = exclusionCounter; - } - - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - if (isHidden(dir)) { - return FileVisitResult.SKIP_SUBTREE; - } - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - if (!Files.isHidden(file)) { - fileIndexer.indexFile(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, file, type, progressReport, exclusionCounter, ignoreCommand); - } - return FileVisitResult.CONTINUE; - } - - /** - *

Overridden method to handle exceptions while visiting files in the analysis.

- * - *

- *

    - *
  • FileSystemLoopException - We show a warning that a symlink loop exists and we skip the file.
    • - *
  • AccessDeniedException for excluded files/directories - We skip the file, as files excluded from the analysis, shouldn't throw access exceptions.
    • - *
- *

- * - * @param file a reference to the file - * @param exc the I/O exception that prevented the file from being visited - * @throws IOException - */ - @Override - public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { - if (exc instanceof FileSystemLoopException) { - LOG.warn("Not indexing due to symlink loop: {}", file.toFile()); - return FileVisitResult.CONTINUE; - } else if (exc instanceof AccessDeniedException && isExcluded(file)) { - return FileVisitResult.CONTINUE; - } - throw exc; - } - - /** - *

Checks if the directory is excluded in the analysis or not. Only the exclusions are checked.

- * - *

The inclusions cannot be checked for directories, since the current implementation of pattern matching is intended only for files.

- * - * @param path The file or directory. - * @return True if file/directory is excluded from the analysis, false otherwise. - */ - private boolean isExcluded(Path path) throws IOException { - Path realAbsoluteFile = path.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize(); - return isExcludedDirectory(moduleExclusionFilters, realAbsoluteFile, inputModuleHierarchy.root().getBaseDir(), module.getBaseDir(), type); - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) { - return FileVisitResult.CONTINUE; - } - - private boolean isHidden(Path path) throws IOException { - if (SystemUtils.IS_OS_WINDOWS) { - try { - DosFileAttributes dosFileAttributes = Files.readAttributes(path, DosFileAttributes.class, LinkOption.NOFOLLOW_LINKS); - return dosFileAttributes.isHidden(); - } catch (UnsupportedOperationException e) { - return path.toFile().isHidden(); - } - } else { - return Files.isHidden(path); - } - } + private static String pluralizeFiles(int count) { + return count == 1 ? "file" : "files"; } - static class ExclusionCounter { - private final AtomicInteger excludedByPatternsCount = new AtomicInteger(0); - private final AtomicInteger excludedByScmCount = new AtomicInteger(0); - - public void increaseByPatternsCount() { - excludedByPatternsCount.incrementAndGet(); - } - - public int getByPatternsCount() { - return excludedByPatternsCount.get(); - } - public void increaseByScmCount() { - excludedByScmCount.incrementAndGet(); - } - public int getByScmCount() { - return excludedByScmCount.get(); - } - } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java new file mode 100644 index 00000000000..54d4f2b8d6c --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java @@ -0,0 +1,230 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.scan.filesystem; + +import java.io.IOException; +import java.nio.file.FileVisitOption; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.api.batch.scm.IgnoreCommand; +import org.sonar.api.batch.scm.ScmProvider; +import org.sonar.api.notifications.AnalysisWarnings; +import org.sonar.scanner.bootstrap.GlobalConfiguration; +import org.sonar.scanner.bootstrap.GlobalServerSettings; +import org.sonar.scanner.fs.InputModuleHierarchy; +import org.sonar.scanner.scan.ModuleConfiguration; +import org.sonar.scanner.scan.ModuleConfigurationProvider; +import org.sonar.scanner.scan.ProjectServerSettings; +import org.sonar.scanner.scan.SonarGlobalPropertiesFilter; +import org.sonar.scanner.scm.ScmConfiguration; +import org.sonar.scanner.util.ProgressReport; + +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; + +public class ProjectFilePreprocessor { + + private static final Logger LOG = LoggerFactory.getLogger(ProjectFilePreprocessor.class); + private static final String TELEMETRY_STEP_NAME = "file.preprocessing"; + + private final AnalysisWarnings analysisWarnings; + private final IgnoreCommand ignoreCommand; + private final boolean useScmExclusion; + private final ScmConfiguration scmConfiguration; + private final InputModuleHierarchy inputModuleHierarchy; + private final GlobalConfiguration globalConfig; + private final GlobalServerSettings globalServerSettings; + private final ProjectServerSettings projectServerSettings; + private final LanguageDetection languageDetection; + private final FilePreprocessor filePreprocessor; + private final ProjectExclusionFilters projectExclusionFilters; + + private final SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter; + + private final Map> mainSourcesByModule = new HashMap<>(); + private final Map> testSourcesByModule = new HashMap<>(); + + private int totalFilesPreprocessed = 0; + + public ProjectFilePreprocessor(AnalysisWarnings analysisWarnings, ScmConfiguration scmConfiguration, InputModuleHierarchy inputModuleHierarchy, + GlobalConfiguration globalConfig, GlobalServerSettings globalServerSettings, ProjectServerSettings projectServerSettings, + LanguageDetection languageDetection, FilePreprocessor filePreprocessor, + ProjectExclusionFilters projectExclusionFilters, SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter) { + this.analysisWarnings = analysisWarnings; + this.scmConfiguration = scmConfiguration; + this.inputModuleHierarchy = inputModuleHierarchy; + this.globalConfig = globalConfig; + this.globalServerSettings = globalServerSettings; + this.projectServerSettings = projectServerSettings; + this.languageDetection = languageDetection; + this.filePreprocessor = filePreprocessor; + this.projectExclusionFilters = projectExclusionFilters; + this.sonarGlobalPropertiesFilter = sonarGlobalPropertiesFilter; + this.ignoreCommand = loadIgnoreCommand(); + this.useScmExclusion = ignoreCommand != null; + } + + public void execute() { + ProgressReport progressReport = new ProgressReport("Report about progress of file preprocessing", + TimeUnit.SECONDS.toMillis(10)); + progressReport.start("Preprocessing files..."); + ExclusionCounter exclusionCounter = new ExclusionCounter(); + + if (useScmExclusion) { + ignoreCommand.init(inputModuleHierarchy.root().getBaseDir().toAbsolutePath()); + processModulesRecursively(inputModuleHierarchy.root(), exclusionCounter); + ignoreCommand.clean(); + } else { + processModulesRecursively(inputModuleHierarchy.root(), exclusionCounter); + } + + int totalLanguagesDetected = languageDetection.getDetectedLanguages().size(); + + progressReport.stop(String.format("%s detected in %s", pluralizeWithCount("language", totalLanguagesDetected), + pluralizeWithCount("preprocessed file", totalFilesPreprocessed))); + + int excludedFileByPatternCount = exclusionCounter.getByPatternsCount(); + if (projectExclusionFilters.hasPattern() || excludedFileByPatternCount > 0) { + if (LOG.isInfoEnabled()) { + LOG.info("{} ignored because of inclusion/exclusion patterns", pluralizeWithCount("file", excludedFileByPatternCount)); + } + } + + int excludedFileByScmCount = exclusionCounter.getByScmCount(); + if (useScmExclusion) { + if (LOG.isInfoEnabled()) { + LOG.info("{} ignored because of scm ignore settings", pluralizeWithCount("file", excludedFileByScmCount)); + } + } + } + + private void processModulesRecursively(DefaultInputModule module, ExclusionCounter exclusionCounter) { + inputModuleHierarchy.children(module).stream().sorted(Comparator.comparing(DefaultInputModule::key)).forEach( + m -> processModulesRecursively(m, exclusionCounter)); + processModule(module, exclusionCounter); + } + + private void processModule(DefaultInputModule module, ExclusionCounter exclusionCounter) { + // Emulate creation of module level settings + ModuleConfiguration moduleConfig = new ModuleConfigurationProvider(sonarGlobalPropertiesFilter).provide(globalConfig, module, globalServerSettings, projectServerSettings); + ModuleExclusionFilters moduleExclusionFilters = new ModuleExclusionFilters(moduleConfig, analysisWarnings); + boolean hasChildModules = !module.definition().getSubProjects().isEmpty(); + boolean hasTests = module.getTestDirsOrFiles().isPresent(); + // Default to index basedir when no sources provided + List mainSourceDirsOrFiles = module.getSourceDirsOrFiles() + .orElseGet(() -> hasChildModules || hasTests ? emptyList() : singletonList(module.getBaseDir().toAbsolutePath())); + List processedSources = processModuleSources(module, moduleExclusionFilters, mainSourceDirsOrFiles, InputFile.Type.MAIN, + exclusionCounter); + mainSourcesByModule.put(module, processedSources); + totalFilesPreprocessed += processedSources.size(); + module.getTestDirsOrFiles().ifPresent(tests -> { + List processedTestSources = processModuleSources(module, moduleExclusionFilters, tests, InputFile.Type.TEST, exclusionCounter); + testSourcesByModule.put(module, processedTestSources); + totalFilesPreprocessed += processedTestSources.size(); + }); + } + + private List processModuleSources(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, List sources, + InputFile.Type type, ExclusionCounter exclusionCounter) { + List processedFiles = new ArrayList<>(); + try { + for (Path dirOrFile : sources) { + if (dirOrFile.toFile().isDirectory()) { + processedFiles.addAll(processDirectory(module, moduleExclusionFilters, dirOrFile, type, exclusionCounter)); + } else { + filePreprocessor.processFile(module, moduleExclusionFilters, dirOrFile, type, exclusionCounter, ignoreCommand) + .ifPresent(processedFiles::add); + } + } + } catch (IOException e) { + throw new IllegalStateException("Failed to preprocess files", e); + } + return processedFiles; + } + + private List processDirectory(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, Path path, + InputFile.Type type, ExclusionCounter exclusionCounter) throws IOException { + List processedFiles = new ArrayList<>(); + Files.walkFileTree(path.normalize(), Collections.singleton(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, + new DirectoryFileVisitor(file -> filePreprocessor.processFile(module, moduleExclusionFilters, file, type, exclusionCounter, + ignoreCommand).ifPresent(processedFiles::add), module, moduleExclusionFilters, inputModuleHierarchy, type) + ); + return processedFiles; + } + + public List getMainSourcesByModule(DefaultInputModule module) { + return Collections.unmodifiableList(mainSourcesByModule.get(module)); + } + + public Optional> getTestSourcesByModule(DefaultInputModule module) { + return Optional.ofNullable(testSourcesByModule.get(module)).map(Collections::unmodifiableList); + } + + private IgnoreCommand loadIgnoreCommand() { + try { + ScmProvider provider = scmConfiguration.provider(); + if (!scmConfiguration.isExclusionDisabled() && provider != null) { + return provider.ignoreCommand(); + } + } catch (UnsupportedOperationException e) { + LOG.debug("File exclusion based on SCM ignore information is not available with this plugin."); + } + + return null; + } + + private static String pluralizeWithCount(String str, int count) { + String pluralized = count == 1 ? str : (str + "s"); + return count + " " + pluralized; + } + + public static class ExclusionCounter { + private int excludedByPatternsCount = 0; + private int excludedByScmCount = 0; + + public void increaseByPatternsCount() { + excludedByPatternsCount++; + } + + public int getByPatternsCount() { + return excludedByPatternsCount; + } + + public void increaseByScmCount() { + excludedByScmCount++; + } + + public int getByScmCount() { + return excludedByScmCount; + } + } +} -- cgit v1.2.3