diff options
author | Matteo Mara <matteo.mara@sonarsource.com> | 2023-12-22 10:28:20 +0100 |
---|---|---|
committer | sonartech <sonartech@sonarsource.com> | 2023-12-22 20:03:01 +0000 |
commit | 48705fd9366d3cfae6c4642084d0400ee73313ad (patch) | |
tree | 1932ef20c1ff00118d8b83a6879e16d62a788381 /sonar-scanner-engine | |
parent | b6367ef978efd8f342dd70c4427e7570320818a4 (diff) | |
download | sonarqube-48705fd9366d3cfae6c4642084d0400ee73313ad.tar.gz sonarqube-48705fd9366d3cfae6c4642084d0400ee73313ad.zip |
Revert "SONAR-21195 Refactor file indexing into two distinct steps"
This reverts commit 42d16dafde01b19bc7fcb99084e087fdc9b8e95e.
Diffstat (limited to 'sonar-scanner-engine')
24 files changed, 355 insertions, 992 deletions
diff --git a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/ScannerMediumTester.java b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/ScannerMediumTester.java index 1916d3e6d00..ebf0dc20efa 100644 --- a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/ScannerMediumTester.java +++ b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/ScannerMediumTester.java @@ -35,7 +35,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Properties; -import java.util.Set; import javax.annotation.CheckForNull; import javax.annotation.Nullable; import javax.annotation.Priority; @@ -137,12 +136,11 @@ public class ScannerMediumTester extends ExternalResource { } public ScannerMediumTester registerPlugin(String pluginKey, Plugin instance) { - pluginInstaller.add(pluginKey, instance); - return this; + return registerPlugin(pluginKey, instance, 1L); } - public ScannerMediumTester registerOptionalPlugin(String pluginKey, Set<String> requiredForLanguages, Plugin instance) { - pluginInstaller.addOptional(pluginKey, requiredForLanguages, instance); + public ScannerMediumTester registerPlugin(String pluginKey, Plugin instance, long lastUpdatedAt) { + pluginInstaller.add(pluginKey, instance, lastUpdatedAt); return this; } diff --git a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java index 66f35f9cfa8..0bfef5a38ad 100644 --- a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java +++ b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java @@ -29,7 +29,6 @@ import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Random; -import java.util.Set; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.SystemUtils; @@ -40,10 +39,8 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.slf4j.event.Level; import org.sonar.api.CoreProperties; -import org.sonar.api.Plugin; import org.sonar.api.SonarEdition; import org.sonar.api.batch.fs.InputFile; -import org.sonar.api.batch.fs.InputFileFilter; import org.sonar.api.batch.fs.internal.DefaultInputFile; import org.sonar.api.testfixtures.log.LogTester; import org.sonar.api.utils.MessageException; @@ -77,7 +74,6 @@ public class FileSystemMediumIT { public ScannerMediumTester tester = new ScannerMediumTester() .setEdition(SonarEdition.COMMUNITY) .registerPlugin("xoo", new XooPlugin()) - .registerOptionalPlugin("optional-xoo", Set.of("xoo"), new OptionalXooPlugin()) .addDefaultQProfile("xoo", "Sonar Way") .addDefaultQProfile("xoo2", "Sonar Way"); @@ -1185,7 +1181,7 @@ public class FileSystemMediumIT { assertThatThrownBy(result::execute) .isExactlyInstanceOf(IllegalStateException.class) - .hasMessageEndingWith(format("Failed to preprocess files")); + .hasMessageEndingWith(format("Failed to index files")); } @Test @@ -1256,42 +1252,7 @@ public class FileSystemMediumIT { assertThatThrownBy(result::execute) .isExactlyInstanceOf(IllegalStateException.class) - .hasMessageEndingWith(format("Failed to preprocess files")); - } - - @Test - public void should_load_input_file_filters_for_required_and_optional_plugins() throws IOException { - File projectDir = new File("test-resources/mediumtest/xoo/sample-with-input-file-filters"); - AnalysisResult result = tester - .newAnalysis(new File(projectDir, "sonar-project.properties")) - .execute(); - - assertThat(result.inputFiles()).hasSize(1); - - assertThat(logTester.logs()).contains("'xources/hello/xoo_exclude2.xoo' excluded by org.sonar.scanner.mediumtest.fs" + - ".FileSystemMediumIT$OptionalXooPlugin$OptionalXooFileFilter"); - assertThat(logTester.logs()).contains("'xources/hello/xoo_exclude.xoo' excluded by org.sonar.xoo.extensions.XooExcludeFileFilter"); - assertThat(logTester.logs()).contains("'xources/hello/HelloJava.xoo' indexed with language 'xoo'"); - - assertThat(result.inputFile("xources/hello/xoo_exclude.xoo")).isNull(); - assertThat(result.inputFile("xources/hello/xoo_exclude2.xoo")).isNull(); - assertThat(result.inputFile("xources/hello/HelloJava.xoo")).isNotNull(); - } - - public static class OptionalXooPlugin implements Plugin { - - @Override - public void define(Context context) { - context.addExtension(OptionalXooFileFilter.class); - } - - public static class OptionalXooFileFilter implements InputFileFilter { - - @Override - public boolean accept(InputFile f) { - return !f.filename().endsWith("_exclude2.xoo"); - } - } + .hasMessageEndingWith(format("Failed to index files")); } private static void assertAnalysedFiles(AnalysisResult result, String... files) { diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java index de69726669c..eeb35462dab 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java @@ -22,7 +22,6 @@ package org.sonar.scanner.bootstrap; import java.util.List; import java.util.Map; import java.util.Set; -import org.sonar.scanner.mediumtest.LocalPlugin; public interface PluginInstaller { @@ -46,14 +45,8 @@ public interface PluginInstaller { Map<String, ScannerPlugin> installPluginsForLanguages(Set<String> languageKeys); /** - * Used only by medium tests. Installs required plugins (phase 1) + * Used only by medium tests. * @see org.sonar.scanner.mediumtest.ScannerMediumTester */ - List<LocalPlugin> installLocals(); - - /** - * Used only by medium tests. Installs optional plugins (phase 2) - * @see org.sonar.scanner.mediumtest.ScannerMediumTester - */ - List<LocalPlugin> installOptionalLocals(Set<String> languageKeys); + List<Object[]> installLocals(); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java index e05b7d08471..efed3403147 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java @@ -36,7 +36,6 @@ import org.sonar.api.utils.log.Loggers; import org.sonar.api.utils.log.Profiler; import org.sonar.core.platform.PluginInfo; import org.sonar.core.plugin.PluginType; -import org.sonar.scanner.mediumtest.LocalPlugin; import org.sonarqube.ws.client.GetRequest; import static java.lang.String.format; @@ -138,15 +137,7 @@ public class ScannerPluginInstaller implements PluginInstaller { * Returns empty on purpose. This method is used only by medium tests. */ @Override - public List<LocalPlugin> installLocals() { - return Collections.emptyList(); - } - - /** - * Returns empty on purpose. This method is used only by medium tests. - */ - @Override - public List<LocalPlugin> installOptionalLocals(Set<String> languageKeys) { + public List<Object[]> installLocals() { return Collections.emptyList(); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java index 074079f974e..3ce0202292b 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java @@ -36,7 +36,6 @@ import org.sonar.core.platform.PluginInfo; import org.sonar.core.platform.PluginJarExploder; import org.sonar.core.platform.PluginRepository; import org.sonar.core.plugin.PluginType; -import org.sonar.scanner.mediumtest.LocalPlugin; import static java.util.stream.Collectors.toMap; import static org.sonar.api.utils.Preconditions.checkState; @@ -80,11 +79,11 @@ public class ScannerPluginRepository implements PluginRepository, Startable { pluginInstancesByKeys = new HashMap<>(loader.load(explodedPluginsByKey)); // this part is only used by medium tests - for (LocalPlugin localPlugin : installer.installLocals()) { - ScannerPlugin scannerPlugin = localPlugin.toScannerPlugin(); - String pluginKey = localPlugin.pluginKey(); - pluginsByKeys.put(pluginKey, scannerPlugin); - pluginInstancesByKeys.put(pluginKey, localPlugin.pluginInstance()); + for (Object[] localPlugin : installer.installLocals()) { + String pluginKey = (String) localPlugin[0]; + PluginInfo pluginInfo = new PluginInfo(pluginKey); + pluginsByKeys.put(pluginKey, new ScannerPlugin(pluginInfo.getKey(), (long) localPlugin[2], PluginType.BUNDLED, pluginInfo)); + pluginInstancesByKeys.put(pluginKey, (Plugin) localPlugin[1]); } keysByClassLoader = new HashMap<>(); @@ -108,15 +107,6 @@ public class ScannerPluginRepository implements PluginRepository, Startable { .collect(toMap(Map.Entry::getKey, e -> pluginJarExploder.explode(e.getValue().getInfo()))); pluginInstancesByKeys.putAll(new HashMap<>(loader.load(explodedPluginsByKey))); - // this part is only used by medium tests - for (LocalPlugin localPlugin : installer.installOptionalLocals(languageKeys)) { - ScannerPlugin scannerPlugin = localPlugin.toScannerPlugin(); - String pluginKey = localPlugin.pluginKey(); - languagePluginsByKeys.put(pluginKey, scannerPlugin); - pluginsByKeys.put(pluginKey, scannerPlugin); - pluginInstancesByKeys.put(pluginKey, localPlugin.pluginInstance()); - } - keysByClassLoader = new HashMap<>(); for (Map.Entry<String, Plugin> e : pluginInstancesByKeys.entrySet()) { keysByClassLoader.put(e.getValue().getClass().getClassLoader(), e.getKey()); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java index 38972c95071..eedc07f9924 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java @@ -112,14 +112,13 @@ import org.sonar.scanner.scan.branch.BranchConfigurationProvider; import org.sonar.scanner.scan.branch.BranchType; import org.sonar.scanner.scan.branch.ProjectBranchesProvider; import org.sonar.scanner.scan.filesystem.DefaultProjectFileSystem; -import org.sonar.scanner.scan.filesystem.FilePreprocessor; +import org.sonar.scanner.scan.filesystem.FileIndexer; import org.sonar.scanner.scan.filesystem.InputComponentStore; import org.sonar.scanner.scan.filesystem.LanguageDetection; import org.sonar.scanner.scan.filesystem.MetadataGenerator; -import org.sonar.scanner.scan.filesystem.ModuleRelativePathWarner; import org.sonar.scanner.scan.filesystem.ProjectCoverageAndDuplicationExclusions; import org.sonar.scanner.scan.filesystem.ProjectExclusionFilters; -import org.sonar.scanner.scan.filesystem.ProjectFilePreprocessor; +import org.sonar.scanner.scan.filesystem.ProjectFileIndexer; import org.sonar.scanner.scan.filesystem.ScannerComponentIdGenerator; import org.sonar.scanner.scan.filesystem.StatusDetection; import org.sonar.scanner.scan.measure.DefaultMetricFinder; @@ -194,9 +193,8 @@ public class SpringScannerContainer extends SpringComponentContainer { LanguageDetection.class, MetadataGenerator.class, FileMetadata.class, - ModuleRelativePathWarner.class, - FilePreprocessor.class, - ProjectFilePreprocessor.class, + FileIndexer.class, + ProjectFileIndexer.class, ProjectExclusionFilters.class, // rules @@ -337,7 +335,7 @@ public class SpringScannerContainer extends SpringComponentContainer { getComponentByType(DeprecatedPropertiesWarningGenerator.class).execute(); - getComponentByType(ProjectFilePreprocessor.class).execute(); + getComponentByType(ProjectFileIndexer.class).index(); new SpringProjectScanContainer(this).execute(); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java index 5aaabab3980..fb48e11b424 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java @@ -36,21 +36,15 @@ import org.sonar.scanner.bootstrap.ScannerPlugin; public class FakePluginInstaller implements PluginInstaller { private final Map<String, ScannerPlugin> pluginsByKeys = new HashMap<>(); - private final List<LocalPlugin> mediumTestPlugins = new ArrayList<>(); - private final List<LocalPlugin> optionalMediumTestPlugins = new ArrayList<>(); + private final List<Object[]> mediumTestPlugins = new ArrayList<>(); public FakePluginInstaller add(String pluginKey, File jarFile, long lastUpdatedAt) { pluginsByKeys.put(pluginKey, new ScannerPlugin(pluginKey, lastUpdatedAt, PluginType.BUNDLED, PluginInfo.create(jarFile))); return this; } - public FakePluginInstaller add(String pluginKey, Plugin instance) { - mediumTestPlugins.add(new LocalPlugin(pluginKey, instance, Set.of())); - return this; - } - - public FakePluginInstaller addOptional(String pluginKey, Set<String> requiredForLanguages, Plugin instance) { - optionalMediumTestPlugins.add(new LocalPlugin(pluginKey, instance, requiredForLanguages)); + public FakePluginInstaller add(String pluginKey, Plugin instance, long lastUpdatedAt) { + mediumTestPlugins.add(new Object[] {pluginKey, instance, lastUpdatedAt}); return this; } @@ -70,14 +64,7 @@ public class FakePluginInstaller implements PluginInstaller { } @Override - public List<LocalPlugin> installLocals() { + public List<Object[]> installLocals() { return mediumTestPlugins; } - - @Override - public List<LocalPlugin> installOptionalLocals(Set<String> languageKeys) { - return optionalMediumTestPlugins.stream() - .filter(plugin -> languageKeys.stream().anyMatch(lang -> plugin.requiredForLanguages().contains(lang))) - .toList(); - } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/LocalPlugin.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/LocalPlugin.java deleted file mode 100644 index 20bd4895973..00000000000 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/LocalPlugin.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2023 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.scanner.mediumtest; - -import java.util.Set; -import org.sonar.api.Plugin; -import org.sonar.core.platform.PluginInfo; -import org.sonar.core.plugin.PluginType; -import org.sonar.scanner.bootstrap.ScannerPlugin; - -public record LocalPlugin(String pluginKey, Plugin pluginInstance, Set<String> requiredForLanguages) { - - public ScannerPlugin toScannerPlugin() { - return new ScannerPlugin(pluginKey, 1L, PluginType.BUNDLED, new PluginInfo(pluginKey)); - } -} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java index b70fde51660..351593e9404 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java @@ -49,10 +49,7 @@ import org.sonar.scanner.qualitygate.QualityGateCheck; import org.sonar.scanner.report.ContextPropertiesPublisher; import org.sonar.scanner.report.ReportPublisher; import org.sonar.scanner.rule.QProfileVerifier; -import org.sonar.scanner.scan.filesystem.FileIndexer; -import org.sonar.scanner.scan.filesystem.InputFileFilterRepository; -import org.sonar.scanner.scan.filesystem.LanguageDetection; -import org.sonar.scanner.scan.filesystem.ProjectFileIndexer; +import org.sonar.scanner.scan.filesystem.InputComponentStore; import org.sonar.scanner.scm.ScmPublisher; import org.sonar.scanner.sensor.ProjectSensorExtensionDictionary; import org.sonar.scanner.sensor.ProjectSensorsExecutor; @@ -72,7 +69,7 @@ public class SpringProjectScanContainer extends SpringComponentContainer { @Override protected void doBeforeStart() { - Set<String> languages = getParentComponentByType(LanguageDetection.class).getDetectedLanguages(); + Set<String> languages = getParentComponentByType(InputComponentStore.class).languages(); installPluginsForLanguages(languages); addScannerComponents(); } @@ -117,12 +114,7 @@ public class SpringProjectScanContainer extends SpringComponentContainer { ProjectSensorExtensionDictionary.class, ProjectSensorsExecutor.class, - AnalysisObservers.class, - - // file system - InputFileFilterRepository.class, - FileIndexer.class, - ProjectFileIndexer.class); + AnalysisObservers.class); } static ExtensionMatcher getScannerProjectExtensionsFilter() { @@ -138,7 +130,6 @@ public class SpringProjectScanContainer extends SpringComponentContainer { protected void doAfterStart() { getParentComponentByType(ScannerMetrics.class).addPluginMetrics(getComponentsByType(Metrics.class)); getComponentByType(ProjectLock.class).tryLock(); - getComponentByType(ProjectFileIndexer.class).index(); GlobalAnalysisMode analysisMode = getComponentByType(GlobalAnalysisMode.class); InputModuleHierarchy tree = getComponentByType(InputModuleHierarchy.class); ScanProperties properties = getComponentByType(ScanProperties.class); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java deleted file mode 100644 index eec0ed9c8d0..00000000000 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2023 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.scanner.scan.filesystem; - -import java.io.IOException; -import java.nio.file.AccessDeniedException; -import java.nio.file.FileSystemLoopException; -import java.nio.file.FileVisitResult; -import java.nio.file.FileVisitor; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.Path; -import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.attribute.DosFileAttributes; -import org.apache.commons.lang.SystemUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.sonar.api.batch.fs.InputFile; -import org.sonar.api.batch.fs.internal.DefaultInputModule; -import org.sonar.scanner.fs.InputModuleHierarchy; - -public class DirectoryFileVisitor implements FileVisitor<Path> { - - private static final Logger LOG = LoggerFactory.getLogger(DirectoryFileVisitor.class); - - private final FileVisitAction fileVisitAction; - private final DefaultInputModule module; - private final ModuleExclusionFilters moduleExclusionFilters; - - private final InputModuleHierarchy inputModuleHierarchy; - private final InputFile.Type type; - - DirectoryFileVisitor(FileVisitAction fileVisitAction, DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, InputModuleHierarchy inputModuleHierarchy, InputFile.Type type) { - this.fileVisitAction = fileVisitAction; - this.module = module; - this.moduleExclusionFilters = moduleExclusionFilters; - this.inputModuleHierarchy = inputModuleHierarchy; - this.type = type; - } - - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - return isHidden(dir) ? FileVisitResult.SKIP_SUBTREE : FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - if (!Files.isHidden(file)) { - fileVisitAction.execute(file); - } - return FileVisitResult.CONTINUE; - } - - /** - * <p>Overridden method to handle exceptions while visiting files in the analysis.</p> - * - * <p> - * <ul> - * <li>FileSystemLoopException - We show a warning that a symlink loop exists and we skip the file.</li> - * <li>AccessDeniedException for excluded files/directories - We skip the file, as files excluded from the analysis, shouldn't throw access exceptions.</li> - * </ul> - * </p> - * - * @param file a reference to the file - * @param exc the I/O exception that prevented the file from being visited - * @throws IOException - */ - @Override - public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { - if (exc instanceof FileSystemLoopException) { - LOG.warn("Not indexing due to symlink loop: {}", file.toFile()); - return FileVisitResult.CONTINUE; - } else if (exc instanceof AccessDeniedException && isExcluded(file)) { - return FileVisitResult.CONTINUE; - } - throw exc; - } - - /** - * <p>Checks if the directory is excluded in the analysis or not. Only the exclusions are checked.</p> - * - * <p>The inclusions cannot be checked for directories, since the current implementation of pattern matching is intended only for files.</p> - * - * @param path The file or directory. - * @return True if file/directory is excluded from the analysis, false otherwise. - */ - private boolean isExcluded(Path path) throws IOException { - Path realAbsoluteFile = path.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize(); - return isExcludedDirectory(moduleExclusionFilters, realAbsoluteFile, inputModuleHierarchy.root().getBaseDir(), module.getBaseDir(), type); - } - - /** - * <p>Checks if the path is a directory that is excluded.</p> - * - * <p>Exclusions patterns are checked both at project and module level.</p> - * - * @param moduleExclusionFilters The exclusion filters. - * @param realAbsoluteFile The path to be checked. - * @param projectBaseDir The project base directory. - * @param moduleBaseDir The module base directory. - * @param type The input file type. - * @return True if path is an excluded directory, false otherwise. - */ - private static boolean isExcludedDirectory(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectBaseDir, Path moduleBaseDir, - InputFile.Type type) { - Path projectRelativePath = projectBaseDir.relativize(realAbsoluteFile); - Path moduleRelativePath = moduleBaseDir.relativize(realAbsoluteFile); - return moduleExclusionFilters.isExcludedAsParentDirectoryOfExcludedChildren(realAbsoluteFile, projectRelativePath, projectBaseDir, type) - || moduleExclusionFilters.isExcludedAsParentDirectoryOfExcludedChildren(realAbsoluteFile, moduleRelativePath, moduleBaseDir, type); - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) { - return FileVisitResult.CONTINUE; - } - - private static boolean isHidden(Path path) throws IOException { - if (SystemUtils.IS_OS_WINDOWS) { - try { - DosFileAttributes dosFileAttributes = Files.readAttributes(path, DosFileAttributes.class, LinkOption.NOFOLLOW_LINKS); - return dosFileAttributes.isHidden(); - } catch (UnsupportedOperationException e) { - return path.toFile().isHidden(); - } - } else { - return Files.isHidden(path); - } - } - - @FunctionalInterface - interface FileVisitAction { - void execute(Path file) throws IOException; - } -} - diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java index 4d264590716..4d9b8c2a377 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java @@ -19,10 +19,14 @@ */ package org.sonar.scanner.scan.filesystem; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; import java.nio.file.Path; import java.util.Arrays; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import java.util.function.BooleanSupplier; +import javax.annotation.Nullable; +import org.apache.commons.io.FilenameUtils; import org.sonar.api.CoreProperties; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.InputFile.Type; @@ -32,7 +36,11 @@ import org.sonar.api.batch.fs.internal.DefaultInputFile; import org.sonar.api.batch.fs.internal.DefaultInputModule; import org.sonar.api.batch.fs.internal.DefaultInputProject; import org.sonar.api.batch.fs.internal.SensorStrategy; +import org.sonar.api.batch.scm.IgnoreCommand; +import org.sonar.api.notifications.AnalysisWarnings; import org.sonar.api.utils.MessageException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.sonar.scanner.issue.ignore.scanner.IssueExclusionsLoader; import org.sonar.scanner.repository.language.Language; import org.sonar.scanner.scan.ScanProperties; @@ -48,7 +56,10 @@ public class FileIndexer { private static final Logger LOG = LoggerFactory.getLogger(FileIndexer.class); + private final AnalysisWarnings analysisWarnings; private final ScanProperties properties; + private final InputFileFilter[] filters; + private final ProjectExclusionFilters projectExclusionFilters; private final ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions; private final IssueExclusionsLoader issueExclusionsLoader; private final MetadataGenerator metadataGenerator; @@ -60,13 +71,15 @@ public class FileIndexer { private final StatusDetection statusDetection; private final ScmChangedFiles scmChangedFiles; - private final ModuleRelativePathWarner moduleRelativePathWarner; - private final InputFileFilterRepository inputFileFilterRepository; + private boolean warnInclusionsAlreadyLogged; + private boolean warnExclusionsAlreadyLogged; + private boolean warnCoverageExclusionsAlreadyLogged; + private boolean warnDuplicationExclusionsAlreadyLogged; public FileIndexer(DefaultInputProject project, ScannerComponentIdGenerator scannerComponentIdGenerator, InputComponentStore componentStore, - ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, IssueExclusionsLoader issueExclusionsLoader, - MetadataGenerator metadataGenerator, SensorStrategy sensorStrategy, LanguageDetection languageDetection, ScanProperties properties, - ScmChangedFiles scmChangedFiles, StatusDetection statusDetection, ModuleRelativePathWarner moduleRelativePathWarner, InputFileFilterRepository inputFileFilterRepository) { + ProjectExclusionFilters projectExclusionFilters, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, IssueExclusionsLoader issueExclusionsLoader, + MetadataGenerator metadataGenerator, SensorStrategy sensorStrategy, LanguageDetection languageDetection, AnalysisWarnings analysisWarnings, ScanProperties properties, + InputFileFilter[] filters, ScmChangedFiles scmChangedFiles, StatusDetection statusDetection) { this.project = project; this.scannerComponentIdGenerator = scannerComponentIdGenerator; this.componentStore = componentStore; @@ -75,23 +88,55 @@ public class FileIndexer { this.metadataGenerator = metadataGenerator; this.sensorStrategy = sensorStrategy; this.langDetection = languageDetection; + this.analysisWarnings = analysisWarnings; this.properties = properties; + this.filters = filters; + this.projectExclusionFilters = projectExclusionFilters; this.scmChangedFiles = scmChangedFiles; this.statusDetection = statusDetection; - this.moduleRelativePathWarner = moduleRelativePathWarner; - this.inputFileFilterRepository = inputFileFilterRepository; } - void indexFile(DefaultInputModule module, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path sourceFile, - Type type, ProgressReport progressReport) { - Path projectRelativePath = project.getBaseDir().relativize(sourceFile); - Path moduleRelativePath = module.getBaseDir().relativize(sourceFile); + void indexFile(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, + Path sourceFile, Type type, ProgressReport progressReport, ProjectFileIndexer.ExclusionCounter exclusionCounter, @Nullable IgnoreCommand ignoreCommand) + throws IOException { + // get case of real file without resolving link + Path realAbsoluteFile = sourceFile.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize(); + Path projectRelativePath = project.getBaseDir().relativize(realAbsoluteFile); + Path moduleRelativePath = module.getBaseDir().relativize(realAbsoluteFile); + boolean included = evaluateInclusionsFilters(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type); + if (!included) { + exclusionCounter.increaseByPatternsCount(); + return; + } + boolean excluded = evaluateExclusionsFilters(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type); + if (excluded) { + exclusionCounter.increaseByPatternsCount(); + return; + } + if (!realAbsoluteFile.startsWith(project.getBaseDir())) { + LOG.warn("File '{}' is ignored. It is not located in project basedir '{}'.", realAbsoluteFile.toAbsolutePath(), project.getBaseDir()); + return; + } + if (!realAbsoluteFile.startsWith(module.getBaseDir())) { + LOG.warn("File '{}' is ignored. It is not located in module basedir '{}'.", realAbsoluteFile.toAbsolutePath(), module.getBaseDir()); + return; + } + + if (Files.exists(realAbsoluteFile) && isFileSizeBiggerThanLimit(realAbsoluteFile)) { + LOG.warn("File '{}' is bigger than {}MB and as consequence is removed from the analysis scope.", realAbsoluteFile.toAbsolutePath(), properties.fileSizeLimit()); + return; + } + + Language language = langDetection.language(realAbsoluteFile, projectRelativePath); - // This should be fast; language should be cached from preprocessing step - Language language = langDetection.language(sourceFile, projectRelativePath); + if (ignoreCommand != null && ignoreCommand.isIgnored(realAbsoluteFile)) { + LOG.debug("File '{}' is excluded by the scm ignore settings.", realAbsoluteFile); + exclusionCounter.increaseByScmCount(); + return; + } DefaultIndexedFile indexedFile = new DefaultIndexedFile( - sourceFile, + realAbsoluteFile, project.key(), projectRelativePath.toString(), moduleRelativePath.toString(), @@ -99,7 +144,7 @@ public class FileIndexer { language != null ? language.key() : null, scannerComponentIdGenerator.getAsInt(), sensorStrategy, - scmChangedFiles.getOldRelativeFilePath(sourceFile) + scmChangedFiles.getOldRelativeFilePath(realAbsoluteFile) ); DefaultInputFile inputFile = new DefaultInputFile(indexedFile, f -> metadataGenerator.setMetadata(module.key(), f, module.getEncoding()), @@ -114,9 +159,7 @@ public class FileIndexer { componentStore.put(module.key(), inputFile); issueExclusionsLoader.addMulticriteriaPatterns(inputFile); String langStr = inputFile.language() != null ? format("with language '%s'", inputFile.language()) : "with no language"; - if (LOG.isDebugEnabled()) { - LOG.debug("'{}' indexed {}{}", projectRelativePath, type == Type.TEST ? "as test " : "", langStr); - } + LOG.debug("'{}' indexed {}{}", projectRelativePath, type == Type.TEST ? "as test " : "", langStr); evaluateCoverageExclusions(moduleCoverageAndDuplicationExclusions, inputFile); evaluateDuplicationExclusions(moduleCoverageAndDuplicationExclusions, inputFile); if (properties.preloadFileMetadata()) { @@ -126,6 +169,42 @@ public class FileIndexer { progressReport.message(count + " " + pluralizeFiles(count) + " indexed... (last one was " + inputFile.getProjectRelativePath() + ")"); } + private boolean evaluateInclusionsFilters(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectRelativePath, Path moduleRelativePath, + InputFile.Type type) { + if (!Arrays.equals(moduleExclusionFilters.getInclusionsConfig(type), projectExclusionFilters.getInclusionsConfig(type))) { + // Module specific configuration + return moduleExclusionFilters.isIncluded(realAbsoluteFile, moduleRelativePath, type); + } + boolean includedByProjectConfiguration = projectExclusionFilters.isIncluded(realAbsoluteFile, projectRelativePath, type); + if (includedByProjectConfiguration) { + return true; + } else if (moduleExclusionFilters.isIncluded(realAbsoluteFile, moduleRelativePath, type)) { + warnOnce( + type == Type.MAIN ? CoreProperties.PROJECT_INCLUSIONS_PROPERTY : CoreProperties.PROJECT_TEST_INCLUSIONS_PROPERTY, + FilenameUtils.normalize(projectRelativePath.toString(), true), () -> warnInclusionsAlreadyLogged, () -> warnInclusionsAlreadyLogged = true); + return true; + } + return false; + } + + private boolean evaluateExclusionsFilters(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectRelativePath, Path moduleRelativePath, + InputFile.Type type) { + if (!Arrays.equals(moduleExclusionFilters.getExclusionsConfig(type), projectExclusionFilters.getExclusionsConfig(type))) { + // Module specific configuration + return moduleExclusionFilters.isExcluded(realAbsoluteFile, moduleRelativePath, type); + } + boolean includedByProjectConfiguration = projectExclusionFilters.isExcluded(realAbsoluteFile, projectRelativePath, type); + if (includedByProjectConfiguration) { + return true; + } else if (moduleExclusionFilters.isExcluded(realAbsoluteFile, moduleRelativePath, type)) { + warnOnce( + type == Type.MAIN ? CoreProperties.PROJECT_EXCLUSIONS_PROPERTY : CoreProperties.PROJECT_TEST_EXCLUSIONS_PROPERTY, + FilenameUtils.normalize(projectRelativePath.toString(), true), () -> warnExclusionsAlreadyLogged, () -> warnExclusionsAlreadyLogged = true); + return true; + } + return false; + } + private void checkIfAlreadyIndexed(DefaultInputFile inputFile) { if (componentStore.inputFile(inputFile.getProjectRelativePath()) != null) { throw MessageException.of("File " + inputFile + " can't be indexed twice. Please check that inclusion/exclusion patterns produce " @@ -150,7 +229,8 @@ public class FileIndexer { if (excludedByProjectConfiguration) { return true; } else if (moduleCoverageAndDuplicationExclusions.isExcludedForCoverage(inputFile)) { - moduleRelativePathWarner.warnOnce(CoreProperties.PROJECT_COVERAGE_EXCLUSIONS_PROPERTY, inputFile.getProjectRelativePath()); + warnOnce(CoreProperties.PROJECT_COVERAGE_EXCLUSIONS_PROPERTY, inputFile.getProjectRelativePath(), () -> warnCoverageExclusionsAlreadyLogged, + () -> warnCoverageExclusionsAlreadyLogged = true); return true; } return false; @@ -173,15 +253,26 @@ public class FileIndexer { if (excludedByProjectConfiguration) { return true; } else if (moduleCoverageAndDuplicationExclusions.isExcludedForDuplication(inputFile)) { - moduleRelativePathWarner.warnOnce(CoreProperties.CPD_EXCLUSIONS, inputFile.getProjectRelativePath()); + warnOnce(CoreProperties.CPD_EXCLUSIONS, inputFile.getProjectRelativePath(), () -> warnDuplicationExclusionsAlreadyLogged, + () -> warnDuplicationExclusionsAlreadyLogged = true); return true; } return false; } + private void warnOnce(String propKey, String filePath, BooleanSupplier alreadyLoggedGetter, Runnable markAsLogged) { + if (!alreadyLoggedGetter.getAsBoolean()) { + String msg = "Specifying module-relative paths at project level in the property '" + propKey + "' is deprecated. " + + "To continue matching files like '" + filePath + "', update this property so that patterns refer to project-relative paths."; + LOG.warn(msg); + analysisWarnings.addUnique(msg); + markAsLogged.run(); + } + } + private boolean accept(InputFile indexedFile) { // InputFileFilter extensions. Might trigger generation of metadata - for (InputFileFilter filter : inputFileFilterRepository.getInputFileFilters()) { + for (InputFileFilter filter : filters) { if (!filter.accept(indexedFile)) { LOG.debug("'{}' excluded by {}", indexedFile, filter.getClass().getName()); return false; @@ -194,5 +285,7 @@ public class FileIndexer { return count == 1 ? "file" : "files"; } - + private boolean isFileSizeBiggerThanLimit(Path filePath) throws IOException { + return Files.size(filePath) > properties.fileSizeLimit() * 1024L * 1024L; + } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java deleted file mode 100644 index 6faa87cc42a..00000000000 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2023 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.scanner.scan.filesystem; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.Optional; -import javax.annotation.CheckForNull; -import org.apache.commons.io.FilenameUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.sonar.api.CoreProperties; -import org.sonar.api.batch.fs.InputFile; -import org.sonar.api.batch.fs.internal.DefaultInputModule; -import org.sonar.api.batch.fs.internal.DefaultInputProject; -import org.sonar.api.batch.scm.IgnoreCommand; -import org.sonar.scanner.scan.ScanProperties; - -public class FilePreprocessor { - - private static final Logger LOG = LoggerFactory.getLogger(FilePreprocessor.class); - - private final ModuleRelativePathWarner moduleRelativePathWarner; - private final DefaultInputProject project; - private final LanguageDetection languageDetection; - private final ProjectExclusionFilters projectExclusionFilters; - private final ScanProperties properties; - - public FilePreprocessor(ModuleRelativePathWarner moduleRelativePathWarner, DefaultInputProject project, - LanguageDetection languageDetection, ProjectExclusionFilters projectExclusionFilters, ScanProperties properties) { - this.moduleRelativePathWarner = moduleRelativePathWarner; - this.project = project; - this.languageDetection = languageDetection; - this.projectExclusionFilters = projectExclusionFilters; - this.properties = properties; - } - - public Optional<Path> processFile(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, Path sourceFile, - InputFile.Type type, ProjectFilePreprocessor.ExclusionCounter exclusionCounter, @CheckForNull IgnoreCommand ignoreCommand) throws IOException { - // get case of real file without resolving link - Path realAbsoluteFile = sourceFile.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize(); - Path projectRelativePath = project.getBaseDir().relativize(realAbsoluteFile); - Path moduleRelativePath = module.getBaseDir().relativize(realAbsoluteFile); - boolean included = isFileIncluded(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type); - if (!included) { - exclusionCounter.increaseByPatternsCount(); - return Optional.empty(); - } - boolean excluded = isFileExcluded(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type); - if (excluded) { - exclusionCounter.increaseByPatternsCount(); - return Optional.empty(); - } - - if (!realAbsoluteFile.startsWith(project.getBaseDir())) { - LOG.warn("File '{}' is ignored. It is not located in project basedir '{}'.", realAbsoluteFile.toAbsolutePath(), project.getBaseDir()); - return Optional.empty(); - } - if (!realAbsoluteFile.startsWith(module.getBaseDir())) { - LOG.warn("File '{}' is ignored. It is not located in module basedir '{}'.", realAbsoluteFile.toAbsolutePath(), module.getBaseDir()); - return Optional.empty(); - } - - if (ignoreCommand != null && ignoreCommand.isIgnored(realAbsoluteFile)) { - LOG.debug("File '{}' is excluded by the scm ignore settings.", realAbsoluteFile); - exclusionCounter.increaseByScmCount(); - return Optional.empty(); - } - - if (Files.exists(realAbsoluteFile) && isFileSizeBiggerThanLimit(realAbsoluteFile)) { - LOG.warn("File '{}' is bigger than {}MB and as consequence is removed from the analysis scope.", realAbsoluteFile.toAbsolutePath(), properties.fileSizeLimit()); - return Optional.empty(); - } - - languageDetection.language(realAbsoluteFile, projectRelativePath); - - return Optional.of(realAbsoluteFile); - } - - private boolean isFileIncluded(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectRelativePath, - Path moduleRelativePath, InputFile.Type type) { - if (!Arrays.equals(moduleExclusionFilters.getInclusionsConfig(type), projectExclusionFilters.getInclusionsConfig(type))) { - return moduleExclusionFilters.isIncluded(realAbsoluteFile, moduleRelativePath, type); - } - boolean includedByProjectConfiguration = projectExclusionFilters.isIncluded(realAbsoluteFile, projectRelativePath, type); - if (includedByProjectConfiguration) { - return true; - } - if (moduleExclusionFilters.isIncluded(realAbsoluteFile, moduleRelativePath, type)) { - moduleRelativePathWarner.warnOnce( - type == InputFile.Type.MAIN ? CoreProperties.PROJECT_INCLUSIONS_PROPERTY : CoreProperties.PROJECT_TEST_INCLUSIONS_PROPERTY, - FilenameUtils.normalize(projectRelativePath.toString(), true)); - return true; - } - return false; - } - - private boolean isFileExcluded(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectRelativePath, - Path moduleRelativePath, InputFile.Type type) { - if (!Arrays.equals(moduleExclusionFilters.getExclusionsConfig(type), projectExclusionFilters.getExclusionsConfig(type))) { - return moduleExclusionFilters.isExcluded(realAbsoluteFile, moduleRelativePath, type); - } - boolean includedByProjectConfiguration = projectExclusionFilters.isExcluded(realAbsoluteFile, projectRelativePath, type); - if (includedByProjectConfiguration) { - return true; - } - if (moduleExclusionFilters.isExcluded(realAbsoluteFile, moduleRelativePath, type)) { - moduleRelativePathWarner.warnOnce( - type == InputFile.Type.MAIN ? CoreProperties.PROJECT_EXCLUSIONS_PROPERTY : CoreProperties.PROJECT_TEST_EXCLUSIONS_PROPERTY, - FilenameUtils.normalize(projectRelativePath.toString(), true)); - return true; - } - return false; - } - - private boolean isFileSizeBiggerThanLimit(Path filePath) throws IOException { - return Files.size(filePath) > properties.fileSizeLimit() * 1024L * 1024L; - } -} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepository.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepository.java deleted file mode 100644 index b6e14af85eb..00000000000 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepository.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2023 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.scanner.scan.filesystem; - -import org.sonar.api.batch.fs.InputFileFilter; - -public class InputFileFilterRepository { - private final InputFileFilter[] inputFileFilters; - - public InputFileFilterRepository(InputFileFilter... inputFileFilters) { - this.inputFileFilters = inputFileFilters; - } - - public InputFileFilter[] getInputFileFilters() { - return inputFileFilters; - } -} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java index 24128133824..5ede0250c08 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java @@ -25,7 +25,6 @@ import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.CheckForNull; @@ -54,9 +53,8 @@ public class LanguageDetection { */ private final Map<Language, PathPattern[]> patternsByLanguage; private final List<Language> languagesToConsider; - private final Map<String, Language> languageCacheByPath; - public LanguageDetection(Configuration settings, LanguagesRepository languages, Map<String, Language> languageCache) { + public LanguageDetection(Configuration settings, LanguagesRepository languages) { Map<Language, PathPattern[]> patternsByLanguageBuilder = new LinkedHashMap<>(); for (Language language : languages.all()) { String[] filePatterns = settings.getStringArray(getFileLangPatternPropKey(language.key())); @@ -71,7 +69,6 @@ public class LanguageDetection { languagesToConsider = List.copyOf(patternsByLanguageBuilder.keySet()); patternsByLanguage = unmodifiableMap(patternsByLanguageBuilder); - languageCacheByPath = languageCache; } private static PathPattern[] getLanguagePatterns(Language language) { @@ -92,16 +89,11 @@ public class LanguageDetection { @CheckForNull Language language(Path absolutePath, Path relativePath) { - Language detectedLanguage = languageCacheByPath.get(absolutePath.toString()); - if (detectedLanguage != null) { - return detectedLanguage; - } - + Language detectedLanguage = null; for (Language language : languagesToConsider) { if (isCandidateForLanguage(absolutePath, relativePath, language)) { if (detectedLanguage == null) { detectedLanguage = language; - languageCacheByPath.put(absolutePath.toString(), language); } else { // Language was already forced by another pattern throw MessageException.of(MessageFormat.format("Language of file ''{0}'' can not be decided as the file matches patterns of both {1} and {2}", @@ -113,10 +105,6 @@ public class LanguageDetection { return detectedLanguage; } - public Set<String> getDetectedLanguages() { - return languageCacheByPath.values().stream().map(Language::key).collect(Collectors.toSet()); - } - private boolean isCandidateForLanguage(Path absolutePath, Path relativePath, Language language) { PathPattern[] patterns = patternsByLanguage.get(language); return patterns != null && Arrays.stream(patterns).anyMatch(pattern -> pattern.match(absolutePath, relativePath, false)); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleRelativePathWarner.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleRelativePathWarner.java deleted file mode 100644 index 766955e438c..00000000000 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleRelativePathWarner.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2023 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.scanner.scan.filesystem; - -import java.util.HashSet; -import java.util.Set; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.sonar.api.notifications.AnalysisWarnings; - -public class ModuleRelativePathWarner { - - private static final Logger LOG = LoggerFactory.getLogger(ModuleRelativePathWarner.class); - private final AnalysisWarnings analysisWarnings; - private final Set<String> previouslyWarnedProps = new HashSet<>(); - - public ModuleRelativePathWarner(AnalysisWarnings analysisWarnings) { - this.analysisWarnings = analysisWarnings; - } - - public void warnOnce(String propKey, String filePath) { - if (!previouslyWarnedProps.contains(propKey)) { - previouslyWarnedProps.add(propKey); - String msg = "Specifying module-relative paths at project level in the property '" + propKey + "' is deprecated. " + - "To continue matching files like '" + filePath + "', update this property so that patterns refer to project-relative paths."; - LOG.warn(msg); - analysisWarnings.addUnique(msg); - } - } -} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java index 5ff7ca1ef17..35d5bf3fdb4 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java @@ -20,22 +20,33 @@ package org.sonar.scanner.scan.filesystem; import java.io.IOException; +import java.nio.file.AccessDeniedException; +import java.nio.file.FileSystemLoopException; import java.nio.file.FileVisitOption; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; import java.nio.file.Files; +import java.nio.file.LinkOption; import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.DosFileAttributes; import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.commons.lang.SystemUtils; +import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.InputFile.Type; import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.api.batch.scm.IgnoreCommand; import org.sonar.api.notifications.AnalysisWarnings; import org.sonar.api.scan.filesystem.PathResolver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.sonar.scanner.bootstrap.GlobalConfiguration; import org.sonar.scanner.bootstrap.GlobalServerSettings; import org.sonar.scanner.fs.InputModuleHierarchy; @@ -43,8 +54,12 @@ import org.sonar.scanner.scan.ModuleConfiguration; import org.sonar.scanner.scan.ModuleConfigurationProvider; import org.sonar.scanner.scan.ProjectServerSettings; import org.sonar.scanner.scan.SonarGlobalPropertiesFilter; +import org.sonar.scanner.scm.ScmConfiguration; import org.sonar.scanner.util.ProgressReport; +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; + /** * Index project input files into {@link InputComponentStore}. */ @@ -54,13 +69,15 @@ public class ProjectFileIndexer { private final ProjectExclusionFilters projectExclusionFilters; private final SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter; private final ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions; + private final ScmConfiguration scmConfiguration; private final InputComponentStore componentStore; private final InputModuleHierarchy inputModuleHierarchy; private final GlobalConfiguration globalConfig; private final GlobalServerSettings globalServerSettings; private final ProjectServerSettings projectServerSettings; private final FileIndexer fileIndexer; - private final ProjectFilePreprocessor projectFilePreprocessor; + private final IgnoreCommand ignoreCommand; + private final boolean useScmExclusion; private final AnalysisWarnings analysisWarnings; private ProgressReport progressReport; @@ -68,8 +85,8 @@ public class ProjectFileIndexer { public ProjectFileIndexer(InputComponentStore componentStore, ProjectExclusionFilters exclusionFilters, SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter, InputModuleHierarchy inputModuleHierarchy, GlobalConfiguration globalConfig, GlobalServerSettings globalServerSettings, ProjectServerSettings projectServerSettings, - FileIndexer fileIndexer, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, - ProjectFilePreprocessor projectFilePreprocessor, AnalysisWarnings analysisWarnings) { + FileIndexer fileIndexer, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, ScmConfiguration scmConfiguration, + AnalysisWarnings analysisWarnings) { this.componentStore = componentStore; this.sonarGlobalPropertiesFilter = sonarGlobalPropertiesFilter; this.inputModuleHierarchy = inputModuleHierarchy; @@ -79,8 +96,10 @@ public class ProjectFileIndexer { this.fileIndexer = fileIndexer; this.projectExclusionFilters = exclusionFilters; this.projectCoverageAndDuplicationExclusions = projectCoverageAndDuplicationExclusions; - this.projectFilePreprocessor = projectFilePreprocessor; + this.scmConfiguration = scmConfiguration; this.analysisWarnings = analysisWarnings; + this.ignoreCommand = loadIgnoreCommand(); + this.useScmExclusion = ignoreCommand != null; } public void index() { @@ -89,22 +108,47 @@ public class ProjectFileIndexer { LOG.info("Project configuration:"); projectExclusionFilters.log(" "); projectCoverageAndDuplicationExclusions.log(" "); + ExclusionCounter exclusionCounter = new ExclusionCounter(); - indexModulesRecursively(inputModuleHierarchy.root()); + if (useScmExclusion) { + ignoreCommand.init(inputModuleHierarchy.root().getBaseDir().toAbsolutePath()); + indexModulesRecursively(inputModuleHierarchy.root(), exclusionCounter); + ignoreCommand.clean(); + } else { + indexModulesRecursively(inputModuleHierarchy.root(), exclusionCounter); + } int totalIndexed = componentStore.inputFiles().size(); progressReport.stop(totalIndexed + " " + pluralizeFiles(totalIndexed) + " indexed"); + int excludedFileByPatternCount = exclusionCounter.getByPatternsCount(); + if (projectExclusionFilters.hasPattern() || excludedFileByPatternCount > 0) { + LOG.info("{} {} ignored because of inclusion/exclusion patterns", excludedFileByPatternCount, pluralizeFiles(excludedFileByPatternCount)); + } + int excludedFileByScmCount = exclusionCounter.getByScmCount(); + if (useScmExclusion) { + LOG.info("{} {} ignored because of scm ignore settings", excludedFileByScmCount, pluralizeFiles(excludedFileByScmCount)); + } } - private void indexModulesRecursively(DefaultInputModule module) { - inputModuleHierarchy.children(module).stream() - .sorted(Comparator.comparing(DefaultInputModule::key)) - .forEach(this::indexModulesRecursively); - index(module); + private IgnoreCommand loadIgnoreCommand() { + try { + if (!scmConfiguration.isExclusionDisabled() && scmConfiguration.provider() != null) { + return scmConfiguration.provider().ignoreCommand(); + } + } catch (UnsupportedOperationException e) { + LOG.debug("File exclusion based on SCM ignore information is not available with this plugin."); + } + + return null; } - private void index(DefaultInputModule module) { + private void indexModulesRecursively(DefaultInputModule module, ExclusionCounter exclusionCounter) { + inputModuleHierarchy.children(module).stream().sorted(Comparator.comparing(DefaultInputModule::key)).forEach(m -> indexModulesRecursively(m, exclusionCounter)); + index(module, exclusionCounter); + } + + private void index(DefaultInputModule module, ExclusionCounter exclusionCounter) { // Emulate creation of module level settings ModuleConfiguration moduleConfig = new ModuleConfigurationProvider(sonarGlobalPropertiesFilter).provide(globalConfig, module, globalServerSettings, projectServerSettings); ModuleExclusionFilters moduleExclusionFilters = new ModuleExclusionFilters(moduleConfig, analysisWarnings); @@ -117,10 +161,13 @@ public class ProjectFileIndexer { moduleExclusionFilters.log(" "); moduleCoverageAndDuplicationExclusions.log(" "); } - List<Path> mainSourceDirsOrFiles = projectFilePreprocessor.getMainSourcesByModule(module); - indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, mainSourceDirsOrFiles, Type.MAIN); - projectFilePreprocessor.getTestSourcesByModule(module) - .ifPresent(tests -> indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, tests, Type.TEST)); + boolean hasChildModules = !module.definition().getSubProjects().isEmpty(); + boolean hasTests = module.getTestDirsOrFiles().isPresent(); + // Default to index basedir when no sources provided + List<Path> mainSourceDirsOrFiles = module.getSourceDirsOrFiles() + .orElseGet(() -> hasChildModules || hasTests ? emptyList() : singletonList(module.getBaseDir().toAbsolutePath())); + indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, mainSourceDirsOrFiles, Type.MAIN, exclusionCounter); + module.getTestDirsOrFiles().ifPresent(tests -> indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, tests, Type.TEST, exclusionCounter)); } private static void logPaths(String label, Path baseDir, List<Path> paths) { @@ -129,7 +176,7 @@ public class ProjectFileIndexer { for (Iterator<Path> it = paths.iterator(); it.hasNext(); ) { Path file = it.next(); Optional<String> relativePathToBaseDir = PathResolver.relativize(baseDir, file); - if (relativePathToBaseDir.isEmpty()) { + if (!relativePathToBaseDir.isPresent()) { sb.append(file); } else if (StringUtils.isBlank(relativePathToBaseDir.get())) { sb.append("."); @@ -148,14 +195,19 @@ public class ProjectFileIndexer { } } - private void indexFiles(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, - List<Path> sources, Type type) { + private static String pluralizeFiles(int count) { + return count == 1 ? "file" : "files"; + } + + private void indexFiles(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, + ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, List<Path> sources, Type type, ExclusionCounter exclusionCounter) { try { for (Path dirOrFile : sources) { if (dirOrFile.toFile().isDirectory()) { - indexDirectory(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type); + indexDirectory(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type, exclusionCounter); } else { - fileIndexer.indexFile(module, moduleCoverageAndDuplicationExclusions, dirOrFile, type, progressReport); + fileIndexer.indexFile(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type, progressReport, exclusionCounter, + ignoreCommand); } } } catch (IOException e) { @@ -164,17 +216,141 @@ public class ProjectFileIndexer { } private void indexDirectory(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, - ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, - Path dirToIndex, Type type) throws IOException { + ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path dirToIndex, Type type, ExclusionCounter exclusionCounter) + throws IOException { Files.walkFileTree(dirToIndex.normalize(), Collections.singleton(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, - new DirectoryFileVisitor(file -> fileIndexer.indexFile(module, moduleCoverageAndDuplicationExclusions, file, type, progressReport), - module, moduleExclusionFilters, inputModuleHierarchy, type)); + new IndexFileVisitor(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, type, exclusionCounter)); } - private static String pluralizeFiles(int count) { - return count == 1 ? "file" : "files"; + + /** + * <p>Checks if the path is a directory that is excluded.</p> + * + * <p>Exclusions patterns are checked both at project and module level.</p> + * + * @param moduleExclusionFilters The exclusion filters. + * @param realAbsoluteFile The path to be checked. + * @param projectBaseDir The project base directory. + * @param moduleBaseDir The module base directory. + * @param type The input file type. + * @return True if path is an excluded directory, false otherwise. + */ + private static boolean isExcludedDirectory(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectBaseDir, Path moduleBaseDir, + InputFile.Type type) { + Path projectRelativePath = projectBaseDir.relativize(realAbsoluteFile); + Path moduleRelativePath = moduleBaseDir.relativize(realAbsoluteFile); + return moduleExclusionFilters.isExcludedAsParentDirectoryOfExcludedChildren(realAbsoluteFile, projectRelativePath, projectBaseDir, type) + || moduleExclusionFilters.isExcludedAsParentDirectoryOfExcludedChildren(realAbsoluteFile, moduleRelativePath, moduleBaseDir, type); + } + + private class IndexFileVisitor implements FileVisitor<Path> { + private final DefaultInputModule module; + private final ModuleExclusionFilters moduleExclusionFilters; + private final ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions; + private final Type type; + private final ExclusionCounter exclusionCounter; + + IndexFileVisitor(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, + Type type, + ExclusionCounter exclusionCounter) { + this.module = module; + this.moduleExclusionFilters = moduleExclusionFilters; + this.moduleCoverageAndDuplicationExclusions = moduleCoverageAndDuplicationExclusions; + this.type = type; + this.exclusionCounter = exclusionCounter; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + if (isHidden(dir)) { + return FileVisitResult.SKIP_SUBTREE; + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (!Files.isHidden(file)) { + fileIndexer.indexFile(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, file, type, progressReport, exclusionCounter, ignoreCommand); + } + return FileVisitResult.CONTINUE; + } + + /** + * <p>Overridden method to handle exceptions while visiting files in the analysis.</p> + * + * <p> + * <ul> + * <li>FileSystemLoopException - We show a warning that a symlink loop exists and we skip the file.</li> + * <li>AccessDeniedException for excluded files/directories - We skip the file, as files excluded from the analysis, shouldn't throw access exceptions.</li> + * </ul> + * </p> + * + * @param file a reference to the file + * @param exc the I/O exception that prevented the file from being visited + * @throws IOException + */ + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { + if (exc instanceof FileSystemLoopException) { + LOG.warn("Not indexing due to symlink loop: {}", file.toFile()); + return FileVisitResult.CONTINUE; + } else if (exc instanceof AccessDeniedException && isExcluded(file)) { + return FileVisitResult.CONTINUE; + } + throw exc; + } + + /** + * <p>Checks if the directory is excluded in the analysis or not. Only the exclusions are checked.</p> + * + * <p>The inclusions cannot be checked for directories, since the current implementation of pattern matching is intended only for files.</p> + * + * @param path The file or directory. + * @return True if file/directory is excluded from the analysis, false otherwise. + */ + private boolean isExcluded(Path path) throws IOException { + Path realAbsoluteFile = path.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize(); + return isExcludedDirectory(moduleExclusionFilters, realAbsoluteFile, inputModuleHierarchy.root().getBaseDir(), module.getBaseDir(), type); + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) { + return FileVisitResult.CONTINUE; + } + + private boolean isHidden(Path path) throws IOException { + if (SystemUtils.IS_OS_WINDOWS) { + try { + DosFileAttributes dosFileAttributes = Files.readAttributes(path, DosFileAttributes.class, LinkOption.NOFOLLOW_LINKS); + return dosFileAttributes.isHidden(); + } catch (UnsupportedOperationException e) { + return path.toFile().isHidden(); + } + } else { + return Files.isHidden(path); + } + } } + static class ExclusionCounter { + private final AtomicInteger excludedByPatternsCount = new AtomicInteger(0); + private final AtomicInteger excludedByScmCount = new AtomicInteger(0); + + public void increaseByPatternsCount() { + excludedByPatternsCount.incrementAndGet(); + } + + public int getByPatternsCount() { + return excludedByPatternsCount.get(); + } + public void increaseByScmCount() { + excludedByScmCount.incrementAndGet(); + } + public int getByScmCount() { + return excludedByScmCount.get(); + } + } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java deleted file mode 100644 index 54d4f2b8d6c..00000000000 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java +++ /dev/null @@ -1,230 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2023 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.scanner.scan.filesystem; - -import java.io.IOException; -import java.nio.file.FileVisitOption; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.TimeUnit; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.sonar.api.batch.fs.InputFile; -import org.sonar.api.batch.fs.internal.DefaultInputModule; -import org.sonar.api.batch.scm.IgnoreCommand; -import org.sonar.api.batch.scm.ScmProvider; -import org.sonar.api.notifications.AnalysisWarnings; -import org.sonar.scanner.bootstrap.GlobalConfiguration; -import org.sonar.scanner.bootstrap.GlobalServerSettings; -import org.sonar.scanner.fs.InputModuleHierarchy; -import org.sonar.scanner.scan.ModuleConfiguration; -import org.sonar.scanner.scan.ModuleConfigurationProvider; -import org.sonar.scanner.scan.ProjectServerSettings; -import org.sonar.scanner.scan.SonarGlobalPropertiesFilter; -import org.sonar.scanner.scm.ScmConfiguration; -import org.sonar.scanner.util.ProgressReport; - -import static java.util.Collections.emptyList; -import static java.util.Collections.singletonList; - -public class ProjectFilePreprocessor { - - private static final Logger LOG = LoggerFactory.getLogger(ProjectFilePreprocessor.class); - private static final String TELEMETRY_STEP_NAME = "file.preprocessing"; - - private final AnalysisWarnings analysisWarnings; - private final IgnoreCommand ignoreCommand; - private final boolean useScmExclusion; - private final ScmConfiguration scmConfiguration; - private final InputModuleHierarchy inputModuleHierarchy; - private final GlobalConfiguration globalConfig; - private final GlobalServerSettings globalServerSettings; - private final ProjectServerSettings projectServerSettings; - private final LanguageDetection languageDetection; - private final FilePreprocessor filePreprocessor; - private final ProjectExclusionFilters projectExclusionFilters; - - private final SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter; - - private final Map<DefaultInputModule, List<Path>> mainSourcesByModule = new HashMap<>(); - private final Map<DefaultInputModule, List<Path>> testSourcesByModule = new HashMap<>(); - - private int totalFilesPreprocessed = 0; - - public ProjectFilePreprocessor(AnalysisWarnings analysisWarnings, ScmConfiguration scmConfiguration, InputModuleHierarchy inputModuleHierarchy, - GlobalConfiguration globalConfig, GlobalServerSettings globalServerSettings, ProjectServerSettings projectServerSettings, - LanguageDetection languageDetection, FilePreprocessor filePreprocessor, - ProjectExclusionFilters projectExclusionFilters, SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter) { - this.analysisWarnings = analysisWarnings; - this.scmConfiguration = scmConfiguration; - this.inputModuleHierarchy = inputModuleHierarchy; - this.globalConfig = globalConfig; - this.globalServerSettings = globalServerSettings; - this.projectServerSettings = projectServerSettings; - this.languageDetection = languageDetection; - this.filePreprocessor = filePreprocessor; - this.projectExclusionFilters = projectExclusionFilters; - this.sonarGlobalPropertiesFilter = sonarGlobalPropertiesFilter; - this.ignoreCommand = loadIgnoreCommand(); - this.useScmExclusion = ignoreCommand != null; - } - - public void execute() { - ProgressReport progressReport = new ProgressReport("Report about progress of file preprocessing", - TimeUnit.SECONDS.toMillis(10)); - progressReport.start("Preprocessing files..."); - ExclusionCounter exclusionCounter = new ExclusionCounter(); - - if (useScmExclusion) { - ignoreCommand.init(inputModuleHierarchy.root().getBaseDir().toAbsolutePath()); - processModulesRecursively(inputModuleHierarchy.root(), exclusionCounter); - ignoreCommand.clean(); - } else { - processModulesRecursively(inputModuleHierarchy.root(), exclusionCounter); - } - - int totalLanguagesDetected = languageDetection.getDetectedLanguages().size(); - - progressReport.stop(String.format("%s detected in %s", pluralizeWithCount("language", totalLanguagesDetected), - pluralizeWithCount("preprocessed file", totalFilesPreprocessed))); - - int excludedFileByPatternCount = exclusionCounter.getByPatternsCount(); - if (projectExclusionFilters.hasPattern() || excludedFileByPatternCount > 0) { - if (LOG.isInfoEnabled()) { - LOG.info("{} ignored because of inclusion/exclusion patterns", pluralizeWithCount("file", excludedFileByPatternCount)); - } - } - - int excludedFileByScmCount = exclusionCounter.getByScmCount(); - if (useScmExclusion) { - if (LOG.isInfoEnabled()) { - LOG.info("{} ignored because of scm ignore settings", pluralizeWithCount("file", excludedFileByScmCount)); - } - } - } - - private void processModulesRecursively(DefaultInputModule module, ExclusionCounter exclusionCounter) { - inputModuleHierarchy.children(module).stream().sorted(Comparator.comparing(DefaultInputModule::key)).forEach( - m -> processModulesRecursively(m, exclusionCounter)); - processModule(module, exclusionCounter); - } - - private void processModule(DefaultInputModule module, ExclusionCounter exclusionCounter) { - // Emulate creation of module level settings - ModuleConfiguration moduleConfig = new ModuleConfigurationProvider(sonarGlobalPropertiesFilter).provide(globalConfig, module, globalServerSettings, projectServerSettings); - ModuleExclusionFilters moduleExclusionFilters = new ModuleExclusionFilters(moduleConfig, analysisWarnings); - boolean hasChildModules = !module.definition().getSubProjects().isEmpty(); - boolean hasTests = module.getTestDirsOrFiles().isPresent(); - // Default to index basedir when no sources provided - List<Path> mainSourceDirsOrFiles = module.getSourceDirsOrFiles() - .orElseGet(() -> hasChildModules || hasTests ? emptyList() : singletonList(module.getBaseDir().toAbsolutePath())); - List<Path> processedSources = processModuleSources(module, moduleExclusionFilters, mainSourceDirsOrFiles, InputFile.Type.MAIN, - exclusionCounter); - mainSourcesByModule.put(module, processedSources); - totalFilesPreprocessed += processedSources.size(); - module.getTestDirsOrFiles().ifPresent(tests -> { - List<Path> processedTestSources = processModuleSources(module, moduleExclusionFilters, tests, InputFile.Type.TEST, exclusionCounter); - testSourcesByModule.put(module, processedTestSources); - totalFilesPreprocessed += processedTestSources.size(); - }); - } - - private List<Path> processModuleSources(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, List<Path> sources, - InputFile.Type type, ExclusionCounter exclusionCounter) { - List<Path> processedFiles = new ArrayList<>(); - try { - for (Path dirOrFile : sources) { - if (dirOrFile.toFile().isDirectory()) { - processedFiles.addAll(processDirectory(module, moduleExclusionFilters, dirOrFile, type, exclusionCounter)); - } else { - filePreprocessor.processFile(module, moduleExclusionFilters, dirOrFile, type, exclusionCounter, ignoreCommand) - .ifPresent(processedFiles::add); - } - } - } catch (IOException e) { - throw new IllegalStateException("Failed to preprocess files", e); - } - return processedFiles; - } - - private List<Path> processDirectory(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, Path path, - InputFile.Type type, ExclusionCounter exclusionCounter) throws IOException { - List<Path> processedFiles = new ArrayList<>(); - Files.walkFileTree(path.normalize(), Collections.singleton(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, - new DirectoryFileVisitor(file -> filePreprocessor.processFile(module, moduleExclusionFilters, file, type, exclusionCounter, - ignoreCommand).ifPresent(processedFiles::add), module, moduleExclusionFilters, inputModuleHierarchy, type) - ); - return processedFiles; - } - - public List<Path> getMainSourcesByModule(DefaultInputModule module) { - return Collections.unmodifiableList(mainSourcesByModule.get(module)); - } - - public Optional<List<Path>> getTestSourcesByModule(DefaultInputModule module) { - return Optional.ofNullable(testSourcesByModule.get(module)).map(Collections::unmodifiableList); - } - - private IgnoreCommand loadIgnoreCommand() { - try { - ScmProvider provider = scmConfiguration.provider(); - if (!scmConfiguration.isExclusionDisabled() && provider != null) { - return provider.ignoreCommand(); - } - } catch (UnsupportedOperationException e) { - LOG.debug("File exclusion based on SCM ignore information is not available with this plugin."); - } - - return null; - } - - private static String pluralizeWithCount(String str, int count) { - String pluralized = count == 1 ? str : (str + "s"); - return count + " " + pluralized; - } - - public static class ExclusionCounter { - private int excludedByPatternsCount = 0; - private int excludedByScmCount = 0; - - public void increaseByPatternsCount() { - excludedByPatternsCount++; - } - - public int getByPatternsCount() { - return excludedByPatternsCount; - } - - public void increaseByScmCount() { - excludedByScmCount++; - } - - public int getByScmCount() { - return excludedByScmCount; - } - } -} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitorTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitorTest.java deleted file mode 100644 index fd79238f65d..00000000000 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitorTest.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2023 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.scanner.scan.filesystem; - -import java.io.File; -import java.io.IOException; -import java.nio.file.FileSystemLoopException; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.Path; -import java.nio.file.attribute.BasicFileAttributes; -import org.apache.commons.lang.SystemUtils; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.sonar.api.batch.fs.InputFile; -import org.sonar.api.batch.fs.internal.DefaultInputModule; -import org.sonar.scanner.fs.InputModuleHierarchy; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.assertThrows; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; - -public class DirectoryFileVisitorTest { - - @ClassRule - public static TemporaryFolder temp = new TemporaryFolder(); - - private final DefaultInputModule module = mock(); - private final ModuleExclusionFilters moduleExclusionFilters = mock(); - private final InputModuleHierarchy inputModuleHierarchy = mock(); - private final InputFile.Type type = mock(); - - @Test - public void visit_hidden_file() throws IOException { - DirectoryFileVisitor.FileVisitAction action = mock(DirectoryFileVisitor.FileVisitAction.class); - - File hidden = temp.newFile(".hidden"); - if (SystemUtils.IS_OS_WINDOWS) { - Files.setAttribute(hidden.toPath(), "dos:hidden", true, LinkOption.NOFOLLOW_LINKS); - } - - - DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleExclusionFilters, inputModuleHierarchy, type); - underTest.visitFile(hidden.toPath(), Files.readAttributes(hidden.toPath(), BasicFileAttributes.class)); - - verify(action, never()).execute(any(Path.class)); - } - - @Test - public void test_visit_file_failed_generic_io_exception() throws IOException { - DirectoryFileVisitor.FileVisitAction action = mock(DirectoryFileVisitor.FileVisitAction.class); - - File file = temp.newFile("failed"); - - DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleExclusionFilters, inputModuleHierarchy, type); - assertThrows(IOException.class, () -> underTest.visitFileFailed(file.toPath(), new IOException())); - } - - @Test - public void test_visit_file_failed_file_system_loop_exception() throws IOException { - DirectoryFileVisitor.FileVisitAction action = mock(DirectoryFileVisitor.FileVisitAction.class); - - File file = temp.newFile("symlink"); - - DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleExclusionFilters, inputModuleHierarchy, type); - FileVisitResult result = underTest.visitFileFailed(file.toPath(), new FileSystemLoopException(file.getPath())); - - assertThat(result).isEqualTo(FileVisitResult.CONTINUE); - } - -} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepositoryTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepositoryTest.java deleted file mode 100644 index 12f1cb0c63a..00000000000 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepositoryTest.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2023 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.scanner.scan.filesystem; - -import org.junit.Test; - -import static org.assertj.core.api.Assertions.assertThat; - -public class InputFileFilterRepositoryTest { - - @Test - public void should_not_return_null_if_initialized_with_no_filters() { - InputFileFilterRepository underTest = new InputFileFilterRepository(); - assertThat(underTest.getInputFileFilters()).isNotNull(); - } - - @Test - public void should_return_filters_from_initialization() { - InputFileFilterRepository underTest = new InputFileFilterRepository(f -> true); - assertThat(underTest.getInputFileFilters()).isNotNull(); - assertThat(underTest.getInputFileFilters()).hasSize(1); - } - -} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java index 18408451557..e5810baba5a 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java @@ -24,8 +24,6 @@ import com.tngtech.java.junit.dataprovider.DataProviderRunner; import com.tngtech.java.junit.dataprovider.UseDataProvider; import java.io.File; import java.nio.file.Paths; -import java.util.HashMap; -import java.util.Map; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -40,11 +38,7 @@ import org.sonar.scanner.repository.language.LanguagesRepository; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.endsWith; import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; @RunWith(DataProviderRunner.class) public class LanguageDetectionTest { @@ -78,7 +72,7 @@ public class LanguageDetectionTest { @Test public void detectLanguageKey_shouldDetectByFileExtension() { LanguagesRepository languages = new FakeLanguagesRepository(new Languages(new MockLanguage("java", "java", "jav"), new MockLanguage("cobol", "cbl", "cob"))); - LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>()); + LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages); assertThat(detectLanguageKey(detection, "Foo.java")).isEqualTo("java"); assertThat(detectLanguageKey(detection, "src/Foo.java")).isEqualTo("java"); @@ -100,7 +94,7 @@ public class LanguageDetectionTest { new MockLanguage("docker", new String[0], new String[] {"*.dockerfile", "*.Dockerfile", "Dockerfile", "Dockerfile.*"}), new MockLanguage("terraform", new String[] {"tf"}, new String[] {".tf"}), new MockLanguage("java", new String[0], new String[] {"**/*Test.java"}))); - LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>()); + LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages); assertThat(detectLanguageKey(detection, fileName)).isEqualTo(expectedLanguageKey); } @@ -123,7 +117,7 @@ public class LanguageDetectionTest { @Test public void detectLanguageKey_shouldNotFailIfNoLanguage() { - LanguageDetection detection = spy(new LanguageDetection(settings.asConfig(), new FakeLanguagesRepository(new Languages()), new HashMap<>())); + LanguageDetection detection = spy(new LanguageDetection(settings.asConfig(), new FakeLanguagesRepository(new Languages()))); assertThat(detectLanguageKey(detection, "Foo.java")).isNull(); } @@ -131,14 +125,14 @@ public class LanguageDetectionTest { public void detectLanguageKey_shouldAllowPluginsToDeclareFileExtensionTwiceForCaseSensitivity() { LanguagesRepository languages = new FakeLanguagesRepository(new Languages(new MockLanguage("abap", "abap", "ABAP"))); - LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>()); + LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages); assertThat(detectLanguageKey(detection, "abc.abap")).isEqualTo("abap"); } @Test public void detectLanguageKey_shouldFailIfConflictingLanguageSuffix() { LanguagesRepository languages = new FakeLanguagesRepository(new Languages(new MockLanguage("xml", "xhtml"), new MockLanguage("web", "xhtml"))); - LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>()); + LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages); assertThatThrownBy(() -> detectLanguageKey(detection, "abc.xhtml")) .isInstanceOf(MessageException.class) .hasMessageContaining("Language of file 'abc.xhtml' can not be decided as the file matches patterns of both ") @@ -152,7 +146,7 @@ public class LanguageDetectionTest { settings.setProperty("sonar.lang.patterns.xml", "xml/**"); settings.setProperty("sonar.lang.patterns.web", "web/**"); - LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>()); + LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages); assertThat(detectLanguageKey(detection, "xml/abc.xhtml")).isEqualTo("xml"); assertThat(detectLanguageKey(detection, "web/abc.xhtml")).isEqualTo("web"); } @@ -163,7 +157,7 @@ public class LanguageDetectionTest { settings.setProperty("sonar.lang.patterns.abap", "*.abap,*.txt"); settings.setProperty("sonar.lang.patterns.cobol", "*.cobol,*.txt"); - LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>()); + LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages); assertThat(detectLanguageKey(detection, "abc.abap")).isEqualTo("abap"); assertThat(detectLanguageKey(detection, "abc.cobol")).isEqualTo("cobol"); @@ -174,19 +168,6 @@ public class LanguageDetectionTest { .hasMessageContaining("sonar.lang.patterns.cobol : *.cobol,*.txt"); } - @Test - public void should_cache_detected_language_by_file_path() { - Map<String, org.sonar.scanner.repository.language.Language> languageCacheSpy = spy(new HashMap<>()); - LanguagesRepository languages = new FakeLanguagesRepository(new Languages( - new MockLanguage("java", "java", "jav"), new MockLanguage("cobol", "cbl", "cob"))); - LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, languageCacheSpy); - - assertThat(detectLanguageKey(detection, "Foo.java")).isEqualTo("java"); - assertThat(detectLanguageKey(detection, "Foo.java")).isEqualTo("java"); - verify(languageCacheSpy, times(1)).put(endsWith("/Foo.java"), any(org.sonar.scanner.repository.language.Language.class)); - verify(languageCacheSpy, times(2)).get(endsWith("/Foo.java")); - } - private String detectLanguageKey(LanguageDetection detection, String path) { org.sonar.scanner.repository.language.Language language = detection.language(new File(temp.getRoot(), path).toPath(), Paths.get(path)); return language != null ? language.key() : null; diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/sonar-project.properties b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/sonar-project.properties deleted file mode 100644 index 57c2c062b0e..00000000000 --- a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/sonar-project.properties +++ /dev/null @@ -1,5 +0,0 @@ -sonar.organization=org1 -sonar.projectKey=sample-with-empty-file -sonar.projectName=Sample With Empty -sonar.projectVersion=0.1-SNAPSHOT -sonar.sources=xources diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/HelloJava.xoo b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/HelloJava.xoo deleted file mode 100644 index ee9bf789a53..00000000000 --- a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/HelloJava.xoo +++ /dev/null @@ -1,8 +0,0 @@ -package hello; - -public class HelloJava { - - public static void main(String[] args) { - System.out.println("Hello"); - } -} diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude.xoo b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude.xoo deleted file mode 100644 index 35965a8484c..00000000000 --- a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude.xoo +++ /dev/null @@ -1 +0,0 @@ -this file should be excluded from indexing. diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude2.xoo b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude2.xoo deleted file mode 100644 index d04e465a561..00000000000 --- a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude2.xoo +++ /dev/null @@ -1 +0,0 @@ -this file should ALSO be excluded from indexing. |