diff options
author | Matteo Mara <matteo.mara@sonarsource.com> | 2023-12-15 14:31:39 +0100 |
---|---|---|
committer | sonartech <sonartech@sonarsource.com> | 2024-01-04 20:02:48 +0000 |
commit | f8465c0d33ceb835c92f0b24e7842ba96c7604e0 (patch) | |
tree | e6b4bf5fec92c471e7510ee92b003fbc49dc24fa | |
parent | aec0c95e903db2ab7fdc3dfc6e90421e6342a277 (diff) | |
download | sonarqube-f8465c0d33ceb835c92f0b24e7842ba96c7604e0.tar.gz sonarqube-f8465c0d33ceb835c92f0b24e7842ba96c7604e0.zip |
SONAR-21195 Refactor file indexing into two distinct steps
27 files changed, 1033 insertions, 357 deletions
diff --git a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java index 5277c977da7..277a14bea00 100644 --- a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java +++ b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java @@ -27,6 +27,7 @@ import org.sonar.api.resources.Qualifiers; import org.sonar.xoo.coverage.ItCoverageSensor; import org.sonar.xoo.coverage.OverallCoverageSensor; import org.sonar.xoo.coverage.UtCoverageSensor; +import org.sonar.xoo.extensions.XooExcludeFileFilter; import org.sonar.xoo.extensions.XooIssueFilter; import org.sonar.xoo.extensions.XooPostJob; import org.sonar.xoo.extensions.XooProjectBuilder; @@ -204,7 +205,8 @@ public class XooPlugin implements Plugin { XooIssueFilter.class, XooIgnoreCommand.class, SignificantCodeSensor.class, - IssueWithCodeVariantsSensor.class); + IssueWithCodeVariantsSensor.class, + XooExcludeFileFilter.class); if (context.getRuntime().getProduct() != SonarProduct.SONARLINT) { context.addExtension(MeasureSensor.class); diff --git a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/extensions/XooExcludeFileFilter.java b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/extensions/XooExcludeFileFilter.java new file mode 100644 index 00000000000..32b31cad93a --- /dev/null +++ b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/extensions/XooExcludeFileFilter.java @@ -0,0 +1,31 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.xoo.extensions; + +import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.batch.fs.InputFileFilter; + +public class XooExcludeFileFilter implements InputFileFilter { + + @Override + public boolean accept(InputFile f) { + return !f.filename().endsWith("_exclude.xoo"); + } +} diff --git a/sonar-core/src/main/java/org/sonar/classloader/ClassloaderBuilder.java b/sonar-core/src/main/java/org/sonar/classloader/ClassloaderBuilder.java index aad8fd69810..076865924dd 100644 --- a/sonar-core/src/main/java/org/sonar/classloader/ClassloaderBuilder.java +++ b/sonar-core/src/main/java/org/sonar/classloader/ClassloaderBuilder.java @@ -117,7 +117,13 @@ public class ClassloaderBuilder { throw new IllegalStateException(String.format("The classloader '%s' already exists in the list of previously created classloaders." + " Can not create it twice.", key)); } - ClassRealm realm = AccessController.<PrivilegedAction<ClassRealm>>doPrivileged(() -> new ClassRealm(key, baseClassloader)); + //TODO: to be checked, the other version of the code is not building + ClassRealm realm = AccessController.doPrivileged(new PrivilegedAction<ClassRealm>() { + @Override + public ClassRealm run() { + return new ClassRealm(key, baseClassloader); + } + }); realm.setStrategy(LoadingOrder.PARENT_FIRST.strategy); newRealmsByKey.put(key, new NewRealm(realm)); return this; diff --git a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/ScannerMediumTester.java b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/ScannerMediumTester.java index 94f1ac31b97..e40c86cf068 100644 --- a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/ScannerMediumTester.java +++ b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/ScannerMediumTester.java @@ -35,6 +35,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Properties; +import java.util.Set; import javax.annotation.CheckForNull; import javax.annotation.Nullable; import javax.annotation.Priority; @@ -136,11 +137,12 @@ public class ScannerMediumTester extends ExternalResource { } public ScannerMediumTester registerPlugin(String pluginKey, Plugin instance) { - return registerPlugin(pluginKey, instance, 1L); + pluginInstaller.add(pluginKey, instance); + return this; } - public ScannerMediumTester registerPlugin(String pluginKey, Plugin instance, long lastUpdatedAt) { - pluginInstaller.add(pluginKey, instance, lastUpdatedAt); + public ScannerMediumTester registerOptionalPlugin(String pluginKey, Set<String> requiredForLanguages, Plugin instance) { + pluginInstaller.addOptional(pluginKey, requiredForLanguages, instance); return this; } diff --git a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java index e06c524d457..1b3ac84f4dc 100644 --- a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java +++ b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java @@ -29,6 +29,7 @@ import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Random; +import java.util.Set; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.SystemUtils; @@ -39,8 +40,10 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.slf4j.event.Level; import org.sonar.api.CoreProperties; +import org.sonar.api.Plugin; import org.sonar.api.SonarEdition; import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.batch.fs.InputFileFilter; import org.sonar.api.batch.fs.internal.DefaultInputFile; import org.sonar.api.testfixtures.log.LogTester; import org.sonar.api.utils.MessageException; @@ -74,6 +77,7 @@ public class FileSystemMediumIT { public ScannerMediumTester tester = new ScannerMediumTester() .setEdition(SonarEdition.COMMUNITY) .registerPlugin("xoo", new XooPlugin()) + .registerOptionalPlugin("optional-xoo", Set.of("xoo"), new OptionalXooPlugin()) .addDefaultQProfile("xoo", "Sonar Way") .addDefaultQProfile("xoo2", "Sonar Way"); @@ -1181,7 +1185,7 @@ public class FileSystemMediumIT { assertThatThrownBy(result::execute) .isExactlyInstanceOf(IllegalStateException.class) - .hasMessageEndingWith(format("Failed to index files")); + .hasMessageEndingWith(format("Failed to preprocess files")); } @Test @@ -1252,7 +1256,42 @@ public class FileSystemMediumIT { assertThatThrownBy(result::execute) .isExactlyInstanceOf(IllegalStateException.class) - .hasMessageEndingWith(format("Failed to index files")); + .hasMessageEndingWith(format("Failed to preprocess files")); + } + + @Test + public void should_load_input_file_filters_for_required_and_optional_plugins() throws IOException { + File projectDir = new File("test-resources/mediumtest/xoo/sample-with-input-file-filters"); + AnalysisResult result = tester + .newAnalysis(new File(projectDir, "sonar-project.properties")) + .execute(); + + assertThat(result.inputFiles()).hasSize(1); + + assertThat(logTester.logs()).contains("'xources/hello/xoo_exclude2.xoo' excluded by org.sonar.scanner.mediumtest.fs" + + ".FileSystemMediumIT$OptionalXooPlugin$OptionalXooFileFilter"); + assertThat(logTester.logs()).contains("'xources/hello/xoo_exclude.xoo' excluded by org.sonar.xoo.extensions.XooExcludeFileFilter"); + assertThat(logTester.logs()).contains("'xources/hello/HelloJava.xoo' indexed with language 'xoo'"); + + assertThat(result.inputFile("xources/hello/xoo_exclude.xoo")).isNull(); + assertThat(result.inputFile("xources/hello/xoo_exclude2.xoo")).isNull(); + assertThat(result.inputFile("xources/hello/HelloJava.xoo")).isNotNull(); + } + + public static class OptionalXooPlugin implements Plugin { + + @Override + public void define(Context context) { + context.addExtension(OptionalXooFileFilter.class); + } + + public static class OptionalXooFileFilter implements InputFileFilter { + + @Override + public boolean accept(InputFile f) { + return !f.filename().endsWith("_exclude2.xoo"); + } + } } private static void assertAnalysedFiles(AnalysisResult result, String... files) { diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java index 0e117e75136..e4ae17968be 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java @@ -22,6 +22,7 @@ package org.sonar.scanner.bootstrap; import java.util.List; import java.util.Map; import java.util.Set; +import org.sonar.scanner.mediumtest.LocalPlugin; public interface PluginInstaller { @@ -45,8 +46,14 @@ public interface PluginInstaller { Map<String, ScannerPlugin> installPluginsForLanguages(Set<String> languageKeys); /** - * Used only by medium tests. + * Used only by medium tests. Installs required plugins (phase 1) * @see org.sonar.scanner.mediumtest.ScannerMediumTester */ - List<Object[]> installLocals(); + List<LocalPlugin> installLocals(); + + /** + * Used only by medium tests. Installs optional plugins (phase 2) + * @see org.sonar.scanner.mediumtest.ScannerMediumTester + */ + List<LocalPlugin> installOptionalLocals(Set<String> languageKeys); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java index d2453c3b6ec..9bf367f630d 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java @@ -36,6 +36,7 @@ import org.sonar.api.utils.log.Loggers; import org.sonar.api.utils.log.Profiler; import org.sonar.core.platform.PluginInfo; import org.sonar.core.plugin.PluginType; +import org.sonar.scanner.mediumtest.LocalPlugin; import org.sonarqube.ws.client.GetRequest; import static java.lang.String.format; @@ -137,7 +138,15 @@ public class ScannerPluginInstaller implements PluginInstaller { * Returns empty on purpose. This method is used only by medium tests. */ @Override - public List<Object[]> installLocals() { + public List<LocalPlugin> installLocals() { + return Collections.emptyList(); + } + + /** + * Returns empty on purpose. This method is used only by medium tests. + */ + @Override + public List<LocalPlugin> installOptionalLocals(Set<String> languageKeys) { return Collections.emptyList(); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java index 4d1894b59f8..6b875da7620 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java @@ -36,6 +36,7 @@ import org.sonar.core.platform.PluginInfo; import org.sonar.core.platform.PluginJarExploder; import org.sonar.core.platform.PluginRepository; import org.sonar.core.plugin.PluginType; +import org.sonar.scanner.mediumtest.LocalPlugin; import static java.util.stream.Collectors.toMap; import static org.sonar.api.utils.Preconditions.checkState; @@ -79,11 +80,11 @@ public class ScannerPluginRepository implements PluginRepository, Startable { pluginInstancesByKeys = new HashMap<>(loader.load(explodedPluginsByKey)); // this part is only used by medium tests - for (Object[] localPlugin : installer.installLocals()) { - String pluginKey = (String) localPlugin[0]; - PluginInfo pluginInfo = new PluginInfo(pluginKey); - pluginsByKeys.put(pluginKey, new ScannerPlugin(pluginInfo.getKey(), (long) localPlugin[2], PluginType.BUNDLED, pluginInfo)); - pluginInstancesByKeys.put(pluginKey, (Plugin) localPlugin[1]); + for (LocalPlugin localPlugin : installer.installLocals()) { + ScannerPlugin scannerPlugin = localPlugin.toScannerPlugin(); + String pluginKey = localPlugin.pluginKey(); + pluginsByKeys.put(pluginKey, scannerPlugin); + pluginInstancesByKeys.put(pluginKey, localPlugin.pluginInstance()); } keysByClassLoader = new HashMap<>(); @@ -107,6 +108,15 @@ public class ScannerPluginRepository implements PluginRepository, Startable { .collect(toMap(Map.Entry::getKey, e -> pluginJarExploder.explode(e.getValue().getInfo()))); pluginInstancesByKeys.putAll(new HashMap<>(loader.load(explodedPluginsByKey))); + // this part is only used by medium tests + for (LocalPlugin localPlugin : installer.installOptionalLocals(languageKeys)) { + ScannerPlugin scannerPlugin = localPlugin.toScannerPlugin(); + String pluginKey = localPlugin.pluginKey(); + languagePluginsByKeys.put(pluginKey, scannerPlugin); + pluginsByKeys.put(pluginKey, scannerPlugin); + pluginInstancesByKeys.put(pluginKey, localPlugin.pluginInstance()); + } + keysByClassLoader = new HashMap<>(); for (Map.Entry<String, Plugin> e : pluginInstancesByKeys.entrySet()) { keysByClassLoader.put(e.getValue().getClass().getClassLoader(), e.getKey()); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java index 0cba8e33dfc..d7cecd16ed3 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java @@ -113,13 +113,14 @@ import org.sonar.scanner.scan.branch.BranchConfigurationProvider; import org.sonar.scanner.scan.branch.BranchType; import org.sonar.scanner.scan.branch.ProjectBranchesProvider; import org.sonar.scanner.scan.filesystem.DefaultProjectFileSystem; -import org.sonar.scanner.scan.filesystem.FileIndexer; +import org.sonar.scanner.scan.filesystem.FilePreprocessor; import org.sonar.scanner.scan.filesystem.InputComponentStore; import org.sonar.scanner.scan.filesystem.LanguageDetection; import org.sonar.scanner.scan.filesystem.MetadataGenerator; +import org.sonar.scanner.scan.filesystem.ModuleRelativePathWarner; import org.sonar.scanner.scan.filesystem.ProjectCoverageAndDuplicationExclusions; import org.sonar.scanner.scan.filesystem.ProjectExclusionFilters; -import org.sonar.scanner.scan.filesystem.ProjectFileIndexer; +import org.sonar.scanner.scan.filesystem.ProjectFilePreprocessor; import org.sonar.scanner.scan.filesystem.ScannerComponentIdGenerator; import org.sonar.scanner.scan.filesystem.StatusDetection; import org.sonar.scanner.scan.measure.DefaultMetricFinder; @@ -194,8 +195,9 @@ public class SpringScannerContainer extends SpringComponentContainer { LanguageDetection.class, MetadataGenerator.class, FileMetadata.class, - FileIndexer.class, - ProjectFileIndexer.class, + ModuleRelativePathWarner.class, + FilePreprocessor.class, + ProjectFilePreprocessor.class, ProjectExclusionFilters.class, // rules @@ -337,7 +339,7 @@ public class SpringScannerContainer extends SpringComponentContainer { getComponentByType(DeprecatedPropertiesWarningGenerator.class).execute(); - getComponentByType(ProjectFileIndexer.class).index(); + getComponentByType(ProjectFilePreprocessor.class).execute(); new SpringProjectScanContainer(this).execute(); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java index 36e2bbbd298..a735827e65d 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java @@ -36,15 +36,21 @@ import org.sonar.scanner.bootstrap.ScannerPlugin; public class FakePluginInstaller implements PluginInstaller { private final Map<String, ScannerPlugin> pluginsByKeys = new HashMap<>(); - private final List<Object[]> mediumTestPlugins = new ArrayList<>(); + private final List<LocalPlugin> mediumTestPlugins = new ArrayList<>(); + private final List<LocalPlugin> optionalMediumTestPlugins = new ArrayList<>(); public FakePluginInstaller add(String pluginKey, File jarFile, long lastUpdatedAt) { pluginsByKeys.put(pluginKey, new ScannerPlugin(pluginKey, lastUpdatedAt, PluginType.BUNDLED, PluginInfo.create(jarFile))); return this; } - public FakePluginInstaller add(String pluginKey, Plugin instance, long lastUpdatedAt) { - mediumTestPlugins.add(new Object[] {pluginKey, instance, lastUpdatedAt}); + public FakePluginInstaller add(String pluginKey, Plugin instance) { + mediumTestPlugins.add(new LocalPlugin(pluginKey, instance, Set.of())); + return this; + } + + public FakePluginInstaller addOptional(String pluginKey, Set<String> requiredForLanguages, Plugin instance) { + optionalMediumTestPlugins.add(new LocalPlugin(pluginKey, instance, requiredForLanguages)); return this; } @@ -64,7 +70,14 @@ public class FakePluginInstaller implements PluginInstaller { } @Override - public List<Object[]> installLocals() { + public List<LocalPlugin> installLocals() { return mediumTestPlugins; } + + @Override + public List<LocalPlugin> installOptionalLocals(Set<String> languageKeys) { + return optionalMediumTestPlugins.stream() + .filter(plugin -> languageKeys.stream().anyMatch(lang -> plugin.requiredForLanguages().contains(lang))) + .toList(); + } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/LocalPlugin.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/LocalPlugin.java new file mode 100644 index 00000000000..dc8f3701e47 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/LocalPlugin.java @@ -0,0 +1,33 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.mediumtest; + +import java.util.Set; +import org.sonar.api.Plugin; +import org.sonar.core.platform.PluginInfo; +import org.sonar.core.plugin.PluginType; +import org.sonar.scanner.bootstrap.ScannerPlugin; + +public record LocalPlugin(String pluginKey, Plugin pluginInstance, Set<String> requiredForLanguages) { + + public ScannerPlugin toScannerPlugin() { + return new ScannerPlugin(pluginKey, 1L, PluginType.BUNDLED, new PluginInfo(pluginKey)); + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java index 28a55fdaf98..97ed1019fdd 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java @@ -48,7 +48,10 @@ import org.sonar.scanner.postjob.PostJobsExecutor; import org.sonar.scanner.qualitygate.QualityGateCheck; import org.sonar.scanner.report.ReportPublisher; import org.sonar.scanner.rule.QProfileVerifier; -import org.sonar.scanner.scan.filesystem.InputComponentStore; +import org.sonar.scanner.scan.filesystem.FileIndexer; +import org.sonar.scanner.scan.filesystem.InputFileFilterRepository; +import org.sonar.scanner.scan.filesystem.LanguageDetection; +import org.sonar.scanner.scan.filesystem.ProjectFileIndexer; import org.sonar.scanner.scm.ScmPublisher; import org.sonar.scanner.sensor.ProjectSensorExtensionDictionary; import org.sonar.scanner.sensor.ProjectSensorsExecutor; @@ -68,7 +71,7 @@ public class SpringProjectScanContainer extends SpringComponentContainer { @Override protected void doBeforeStart() { - Set<String> languages = getParentComponentByType(InputComponentStore.class).languages(); + Set<String> languages = getParentComponentByType(LanguageDetection.class).getDetectedLanguages(); installPluginsForLanguages(languages); addScannerComponents(); } @@ -111,7 +114,12 @@ public class SpringProjectScanContainer extends SpringComponentContainer { ProjectSensorExtensionDictionary.class, ProjectSensorsExecutor.class, - AnalysisObservers.class); + AnalysisObservers.class, + + // file system + InputFileFilterRepository.class, + FileIndexer.class, + ProjectFileIndexer.class); } static ExtensionMatcher getScannerProjectExtensionsFilter() { @@ -127,6 +135,7 @@ public class SpringProjectScanContainer extends SpringComponentContainer { protected void doAfterStart() { getParentComponentByType(ScannerMetrics.class).addPluginMetrics(getComponentsByType(Metrics.class)); getComponentByType(ProjectLock.class).tryLock(); + getComponentByType(ProjectFileIndexer.class).index(); GlobalAnalysisMode analysisMode = getComponentByType(GlobalAnalysisMode.class); InputModuleHierarchy tree = getComponentByType(InputModuleHierarchy.class); ScanProperties properties = getComponentByType(ScanProperties.class); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java new file mode 100644 index 00000000000..57720692eb0 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java @@ -0,0 +1,152 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.scan.filesystem; + +import java.io.IOException; +import java.nio.file.AccessDeniedException; +import java.nio.file.FileSystemLoopException; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.DosFileAttributes; +import org.apache.commons.lang.SystemUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.scanner.fs.InputModuleHierarchy; + +public class DirectoryFileVisitor implements FileVisitor<Path> { + + private static final Logger LOG = LoggerFactory.getLogger(DirectoryFileVisitor.class); + + private final FileVisitAction fileVisitAction; + private final DefaultInputModule module; + private final ModuleExclusionFilters moduleExclusionFilters; + + private final InputModuleHierarchy inputModuleHierarchy; + private final InputFile.Type type; + + DirectoryFileVisitor(FileVisitAction fileVisitAction, DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, InputModuleHierarchy inputModuleHierarchy, InputFile.Type type) { + this.fileVisitAction = fileVisitAction; + this.module = module; + this.moduleExclusionFilters = moduleExclusionFilters; + this.inputModuleHierarchy = inputModuleHierarchy; + this.type = type; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + return isHidden(dir) ? FileVisitResult.SKIP_SUBTREE : FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (!Files.isHidden(file)) { + fileVisitAction.execute(file); + } + return FileVisitResult.CONTINUE; + } + + /** + * <p>Overridden method to handle exceptions while visiting files in the analysis.</p> + * + * <p> + * <ul> + * <li>FileSystemLoopException - We show a warning that a symlink loop exists and we skip the file.</li> + * <li>AccessDeniedException for excluded files/directories - We skip the file, as files excluded from the analysis, shouldn't throw access exceptions.</li> + * </ul> + * </p> + * + * @param file a reference to the file + * @param exc the I/O exception that prevented the file from being visited + * @throws IOException + */ + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { + if (exc instanceof FileSystemLoopException) { + LOG.warn("Not indexing due to symlink loop: {}", file.toFile()); + return FileVisitResult.CONTINUE; + } else if (exc instanceof AccessDeniedException && isExcluded(file)) { + return FileVisitResult.CONTINUE; + } + throw exc; + } + + /** + * <p>Checks if the directory is excluded in the analysis or not. Only the exclusions are checked.</p> + * + * <p>The inclusions cannot be checked for directories, since the current implementation of pattern matching is intended only for files.</p> + * + * @param path The file or directory. + * @return True if file/directory is excluded from the analysis, false otherwise. + */ + private boolean isExcluded(Path path) throws IOException { + Path realAbsoluteFile = path.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize(); + return isExcludedDirectory(moduleExclusionFilters, realAbsoluteFile, inputModuleHierarchy.root().getBaseDir(), module.getBaseDir(), type); + } + + /** + * <p>Checks if the path is a directory that is excluded.</p> + * + * <p>Exclusions patterns are checked both at project and module level.</p> + * + * @param moduleExclusionFilters The exclusion filters. + * @param realAbsoluteFile The path to be checked. + * @param projectBaseDir The project base directory. + * @param moduleBaseDir The module base directory. + * @param type The input file type. + * @return True if path is an excluded directory, false otherwise. + */ + private static boolean isExcludedDirectory(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectBaseDir, Path moduleBaseDir, + InputFile.Type type) { + Path projectRelativePath = projectBaseDir.relativize(realAbsoluteFile); + Path moduleRelativePath = moduleBaseDir.relativize(realAbsoluteFile); + return moduleExclusionFilters.isExcludedAsParentDirectoryOfExcludedChildren(realAbsoluteFile, projectRelativePath, projectBaseDir, type) + || moduleExclusionFilters.isExcludedAsParentDirectoryOfExcludedChildren(realAbsoluteFile, moduleRelativePath, moduleBaseDir, type); + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) { + return FileVisitResult.CONTINUE; + } + + private static boolean isHidden(Path path) throws IOException { + if (SystemUtils.IS_OS_WINDOWS) { + try { + DosFileAttributes dosFileAttributes = Files.readAttributes(path, DosFileAttributes.class, LinkOption.NOFOLLOW_LINKS); + return dosFileAttributes.isHidden(); + } catch (UnsupportedOperationException e) { + return path.toFile().isHidden(); + } + } else { + return Files.isHidden(path); + } + } + + @FunctionalInterface + interface FileVisitAction { + void execute(Path file) throws IOException; + } +} + diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java index a5c835aea7a..30687416d14 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java @@ -19,14 +19,10 @@ */ package org.sonar.scanner.scan.filesystem; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.LinkOption; import java.nio.file.Path; import java.util.Arrays; -import java.util.function.BooleanSupplier; -import javax.annotation.Nullable; -import org.apache.commons.io.FilenameUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.sonar.api.CoreProperties; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.InputFile.Type; @@ -36,11 +32,7 @@ import org.sonar.api.batch.fs.internal.DefaultInputFile; import org.sonar.api.batch.fs.internal.DefaultInputModule; import org.sonar.api.batch.fs.internal.DefaultInputProject; import org.sonar.api.batch.fs.internal.SensorStrategy; -import org.sonar.api.batch.scm.IgnoreCommand; -import org.sonar.api.notifications.AnalysisWarnings; import org.sonar.api.utils.MessageException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.sonar.scanner.issue.ignore.scanner.IssueExclusionsLoader; import org.sonar.scanner.repository.language.Language; import org.sonar.scanner.scan.ScanProperties; @@ -56,10 +48,7 @@ public class FileIndexer { private static final Logger LOG = LoggerFactory.getLogger(FileIndexer.class); - private final AnalysisWarnings analysisWarnings; private final ScanProperties properties; - private final InputFileFilter[] filters; - private final ProjectExclusionFilters projectExclusionFilters; private final ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions; private final IssueExclusionsLoader issueExclusionsLoader; private final MetadataGenerator metadataGenerator; @@ -71,15 +60,13 @@ public class FileIndexer { private final StatusDetection statusDetection; private final ScmChangedFiles scmChangedFiles; - private boolean warnInclusionsAlreadyLogged; - private boolean warnExclusionsAlreadyLogged; - private boolean warnCoverageExclusionsAlreadyLogged; - private boolean warnDuplicationExclusionsAlreadyLogged; + private final ModuleRelativePathWarner moduleRelativePathWarner; + private final InputFileFilterRepository inputFileFilterRepository; public FileIndexer(DefaultInputProject project, ScannerComponentIdGenerator scannerComponentIdGenerator, InputComponentStore componentStore, - ProjectExclusionFilters projectExclusionFilters, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, IssueExclusionsLoader issueExclusionsLoader, - MetadataGenerator metadataGenerator, SensorStrategy sensorStrategy, LanguageDetection languageDetection, AnalysisWarnings analysisWarnings, ScanProperties properties, - InputFileFilter[] filters, ScmChangedFiles scmChangedFiles, StatusDetection statusDetection) { + ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, IssueExclusionsLoader issueExclusionsLoader, + MetadataGenerator metadataGenerator, SensorStrategy sensorStrategy, LanguageDetection languageDetection, ScanProperties properties, + ScmChangedFiles scmChangedFiles, StatusDetection statusDetection, ModuleRelativePathWarner moduleRelativePathWarner, InputFileFilterRepository inputFileFilterRepository) { this.project = project; this.scannerComponentIdGenerator = scannerComponentIdGenerator; this.componentStore = componentStore; @@ -88,55 +75,23 @@ public class FileIndexer { this.metadataGenerator = metadataGenerator; this.sensorStrategy = sensorStrategy; this.langDetection = languageDetection; - this.analysisWarnings = analysisWarnings; this.properties = properties; - this.filters = filters; - this.projectExclusionFilters = projectExclusionFilters; this.scmChangedFiles = scmChangedFiles; this.statusDetection = statusDetection; + this.moduleRelativePathWarner = moduleRelativePathWarner; + this.inputFileFilterRepository = inputFileFilterRepository; } - void indexFile(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, - Path sourceFile, Type type, ProgressReport progressReport, ProjectFileIndexer.ExclusionCounter exclusionCounter, @Nullable IgnoreCommand ignoreCommand) - throws IOException { - // get case of real file without resolving link - Path realAbsoluteFile = sourceFile.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize(); - Path projectRelativePath = project.getBaseDir().relativize(realAbsoluteFile); - Path moduleRelativePath = module.getBaseDir().relativize(realAbsoluteFile); - boolean included = evaluateInclusionsFilters(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type); - if (!included) { - exclusionCounter.increaseByPatternsCount(); - return; - } - boolean excluded = evaluateExclusionsFilters(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type); - if (excluded) { - exclusionCounter.increaseByPatternsCount(); - return; - } - if (!realAbsoluteFile.startsWith(project.getBaseDir())) { - LOG.warn("File '{}' is ignored. It is not located in project basedir '{}'.", realAbsoluteFile.toAbsolutePath(), project.getBaseDir()); - return; - } - if (!realAbsoluteFile.startsWith(module.getBaseDir())) { - LOG.warn("File '{}' is ignored. It is not located in module basedir '{}'.", realAbsoluteFile.toAbsolutePath(), module.getBaseDir()); - return; - } - - if (Files.exists(realAbsoluteFile) && isFileSizeBiggerThanLimit(realAbsoluteFile)) { - LOG.warn("File '{}' is bigger than {}MB and as consequence is removed from the analysis scope.", realAbsoluteFile.toAbsolutePath(), properties.fileSizeLimit()); - return; - } - - Language language = langDetection.language(realAbsoluteFile, projectRelativePath); + void indexFile(DefaultInputModule module, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path sourceFile, + Type type, ProgressReport progressReport) { + Path projectRelativePath = project.getBaseDir().relativize(sourceFile); + Path moduleRelativePath = module.getBaseDir().relativize(sourceFile); - if (ignoreCommand != null && ignoreCommand.isIgnored(realAbsoluteFile)) { - LOG.debug("File '{}' is excluded by the scm ignore settings.", realAbsoluteFile); - exclusionCounter.increaseByScmCount(); - return; - } + // This should be fast; language should be cached from preprocessing step + Language language = langDetection.language(sourceFile, projectRelativePath); DefaultIndexedFile indexedFile = new DefaultIndexedFile( - realAbsoluteFile, + sourceFile, project.key(), projectRelativePath.toString(), moduleRelativePath.toString(), @@ -144,7 +99,7 @@ public class FileIndexer { language != null ? language.key() : null, scannerComponentIdGenerator.getAsInt(), sensorStrategy, - scmChangedFiles.getOldRelativeFilePath(realAbsoluteFile) + scmChangedFiles.getOldRelativeFilePath(sourceFile) ); DefaultInputFile inputFile = new DefaultInputFile(indexedFile, f -> metadataGenerator.setMetadata(module.key(), f, module.getEncoding()), @@ -159,7 +114,9 @@ public class FileIndexer { componentStore.put(module.key(), inputFile); issueExclusionsLoader.addMulticriteriaPatterns(inputFile); String langStr = inputFile.language() != null ? format("with language '%s'", inputFile.language()) : "with no language"; - LOG.debug("'{}' indexed {}{}", projectRelativePath, type == Type.TEST ? "as test " : "", langStr); + if (LOG.isDebugEnabled()) { + LOG.debug("'{}' indexed {}{}", projectRelativePath, type == Type.TEST ? "as test " : "", langStr); + } evaluateCoverageExclusions(moduleCoverageAndDuplicationExclusions, inputFile); evaluateDuplicationExclusions(moduleCoverageAndDuplicationExclusions, inputFile); if (properties.preloadFileMetadata()) { @@ -169,42 +126,6 @@ public class FileIndexer { progressReport.message(count + " " + pluralizeFiles(count) + " indexed... (last one was " + inputFile.getProjectRelativePath() + ")"); } - private boolean evaluateInclusionsFilters(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectRelativePath, Path moduleRelativePath, - InputFile.Type type) { - if (!Arrays.equals(moduleExclusionFilters.getInclusionsConfig(type), projectExclusionFilters.getInclusionsConfig(type))) { - // Module specific configuration - return moduleExclusionFilters.isIncluded(realAbsoluteFile, moduleRelativePath, type); - } - boolean includedByProjectConfiguration = projectExclusionFilters.isIncluded(realAbsoluteFile, projectRelativePath, type); - if (includedByProjectConfiguration) { - return true; - } else if (moduleExclusionFilters.isIncluded(realAbsoluteFile, moduleRelativePath, type)) { - warnOnce( - type == Type.MAIN ? CoreProperties.PROJECT_INCLUSIONS_PROPERTY : CoreProperties.PROJECT_TEST_INCLUSIONS_PROPERTY, - FilenameUtils.normalize(projectRelativePath.toString(), true), () -> warnInclusionsAlreadyLogged, () -> warnInclusionsAlreadyLogged = true); - return true; - } - return false; - } - - private boolean evaluateExclusionsFilters(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectRelativePath, Path moduleRelativePath, - InputFile.Type type) { - if (!Arrays.equals(moduleExclusionFilters.getExclusionsConfig(type), projectExclusionFilters.getExclusionsConfig(type))) { - // Module specific configuration - return moduleExclusionFilters.isExcluded(realAbsoluteFile, moduleRelativePath, type); - } - boolean includedByProjectConfiguration = projectExclusionFilters.isExcluded(realAbsoluteFile, projectRelativePath, type); - if (includedByProjectConfiguration) { - return true; - } else if (moduleExclusionFilters.isExcluded(realAbsoluteFile, moduleRelativePath, type)) { - warnOnce( - type == Type.MAIN ? CoreProperties.PROJECT_EXCLUSIONS_PROPERTY : CoreProperties.PROJECT_TEST_EXCLUSIONS_PROPERTY, - FilenameUtils.normalize(projectRelativePath.toString(), true), () -> warnExclusionsAlreadyLogged, () -> warnExclusionsAlreadyLogged = true); - return true; - } - return false; - } - private void checkIfAlreadyIndexed(DefaultInputFile inputFile) { if (componentStore.inputFile(inputFile.getProjectRelativePath()) != null) { throw MessageException.of("File " + inputFile + " can't be indexed twice. Please check that inclusion/exclusion patterns produce " @@ -229,8 +150,7 @@ public class FileIndexer { if (excludedByProjectConfiguration) { return true; } else if (moduleCoverageAndDuplicationExclusions.isExcludedForCoverage(inputFile)) { - warnOnce(CoreProperties.PROJECT_COVERAGE_EXCLUSIONS_PROPERTY, inputFile.getProjectRelativePath(), () -> warnCoverageExclusionsAlreadyLogged, - () -> warnCoverageExclusionsAlreadyLogged = true); + moduleRelativePathWarner.warnOnce(CoreProperties.PROJECT_COVERAGE_EXCLUSIONS_PROPERTY, inputFile.getProjectRelativePath()); return true; } return false; @@ -253,26 +173,15 @@ public class FileIndexer { if (excludedByProjectConfiguration) { return true; } else if (moduleCoverageAndDuplicationExclusions.isExcludedForDuplication(inputFile)) { - warnOnce(CoreProperties.CPD_EXCLUSIONS, inputFile.getProjectRelativePath(), () -> warnDuplicationExclusionsAlreadyLogged, - () -> warnDuplicationExclusionsAlreadyLogged = true); + moduleRelativePathWarner.warnOnce(CoreProperties.CPD_EXCLUSIONS, inputFile.getProjectRelativePath()); return true; } return false; } - private void warnOnce(String propKey, String filePath, BooleanSupplier alreadyLoggedGetter, Runnable markAsLogged) { - if (!alreadyLoggedGetter.getAsBoolean()) { - String msg = "Specifying module-relative paths at project level in the property '" + propKey + "' is deprecated. " + - "To continue matching files like '" + filePath + "', update this property so that patterns refer to project-relative paths."; - LOG.warn(msg); - analysisWarnings.addUnique(msg); - markAsLogged.run(); - } - } - private boolean accept(InputFile indexedFile) { // InputFileFilter extensions. Might trigger generation of metadata - for (InputFileFilter filter : filters) { + for (InputFileFilter filter : inputFileFilterRepository.getInputFileFilters()) { if (!filter.accept(indexedFile)) { LOG.debug("'{}' excluded by {}", indexedFile, filter.getClass().getName()); return false; @@ -285,7 +194,5 @@ public class FileIndexer { return count == 1 ? "file" : "files"; } - private boolean isFileSizeBiggerThanLimit(Path filePath) throws IOException { - return Files.size(filePath) > properties.fileSizeLimit() * 1024L * 1024L; - } + } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java new file mode 100644 index 00000000000..d65c27975ab --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java @@ -0,0 +1,139 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.scan.filesystem; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Optional; +import javax.annotation.CheckForNull; +import org.apache.commons.io.FilenameUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.CoreProperties; +import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.api.batch.fs.internal.DefaultInputProject; +import org.sonar.api.batch.scm.IgnoreCommand; +import org.sonar.scanner.scan.ScanProperties; + +public class FilePreprocessor { + + private static final Logger LOG = LoggerFactory.getLogger(FilePreprocessor.class); + + private final ModuleRelativePathWarner moduleRelativePathWarner; + private final DefaultInputProject project; + private final LanguageDetection languageDetection; + private final ProjectExclusionFilters projectExclusionFilters; + private final ScanProperties properties; + + public FilePreprocessor(ModuleRelativePathWarner moduleRelativePathWarner, DefaultInputProject project, + LanguageDetection languageDetection, ProjectExclusionFilters projectExclusionFilters, ScanProperties properties) { + this.moduleRelativePathWarner = moduleRelativePathWarner; + this.project = project; + this.languageDetection = languageDetection; + this.projectExclusionFilters = projectExclusionFilters; + this.properties = properties; + } + + public Optional<Path> processFile(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, Path sourceFile, + InputFile.Type type, ProjectFilePreprocessor.ExclusionCounter exclusionCounter, @CheckForNull IgnoreCommand ignoreCommand) throws IOException { + // get case of real file without resolving link + Path realAbsoluteFile = sourceFile.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize(); + Path projectRelativePath = project.getBaseDir().relativize(realAbsoluteFile); + Path moduleRelativePath = module.getBaseDir().relativize(realAbsoluteFile); + boolean included = isFileIncluded(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type); + if (!included) { + exclusionCounter.increaseByPatternsCount(); + return Optional.empty(); + } + boolean excluded = isFileExcluded(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type); + if (excluded) { + exclusionCounter.increaseByPatternsCount(); + return Optional.empty(); + } + + if (!realAbsoluteFile.startsWith(project.getBaseDir())) { + LOG.warn("File '{}' is ignored. It is not located in project basedir '{}'.", realAbsoluteFile.toAbsolutePath(), project.getBaseDir()); + return Optional.empty(); + } + if (!realAbsoluteFile.startsWith(module.getBaseDir())) { + LOG.warn("File '{}' is ignored. It is not located in module basedir '{}'.", realAbsoluteFile.toAbsolutePath(), module.getBaseDir()); + return Optional.empty(); + } + + if (ignoreCommand != null && ignoreCommand.isIgnored(realAbsoluteFile)) { + LOG.debug("File '{}' is excluded by the scm ignore settings.", realAbsoluteFile); + exclusionCounter.increaseByScmCount(); + return Optional.empty(); + } + + if (Files.exists(realAbsoluteFile) && isFileSizeBiggerThanLimit(realAbsoluteFile)) { + LOG.warn("File '{}' is bigger than {}MB and as consequence is removed from the analysis scope.", realAbsoluteFile.toAbsolutePath(), properties.fileSizeLimit()); + return Optional.empty(); + } + + languageDetection.language(realAbsoluteFile, projectRelativePath); + + return Optional.of(realAbsoluteFile); + } + + private boolean isFileIncluded(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectRelativePath, + Path moduleRelativePath, InputFile.Type type) { + if (!Arrays.equals(moduleExclusionFilters.getInclusionsConfig(type), projectExclusionFilters.getInclusionsConfig(type))) { + return moduleExclusionFilters.isIncluded(realAbsoluteFile, moduleRelativePath, type); + } + boolean includedByProjectConfiguration = projectExclusionFilters.isIncluded(realAbsoluteFile, projectRelativePath, type); + if (includedByProjectConfiguration) { + return true; + } + if (moduleExclusionFilters.isIncluded(realAbsoluteFile, moduleRelativePath, type)) { + moduleRelativePathWarner.warnOnce( + type == InputFile.Type.MAIN ? CoreProperties.PROJECT_INCLUSIONS_PROPERTY : CoreProperties.PROJECT_TEST_INCLUSIONS_PROPERTY, + FilenameUtils.normalize(projectRelativePath.toString(), true)); + return true; + } + return false; + } + + private boolean isFileExcluded(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectRelativePath, + Path moduleRelativePath, InputFile.Type type) { + if (!Arrays.equals(moduleExclusionFilters.getExclusionsConfig(type), projectExclusionFilters.getExclusionsConfig(type))) { + return moduleExclusionFilters.isExcluded(realAbsoluteFile, moduleRelativePath, type); + } + boolean includedByProjectConfiguration = projectExclusionFilters.isExcluded(realAbsoluteFile, projectRelativePath, type); + if (includedByProjectConfiguration) { + return true; + } + if (moduleExclusionFilters.isExcluded(realAbsoluteFile, moduleRelativePath, type)) { + moduleRelativePathWarner.warnOnce( + type == InputFile.Type.MAIN ? CoreProperties.PROJECT_EXCLUSIONS_PROPERTY : CoreProperties.PROJECT_TEST_EXCLUSIONS_PROPERTY, + FilenameUtils.normalize(projectRelativePath.toString(), true)); + return true; + } + return false; + } + + private boolean isFileSizeBiggerThanLimit(Path filePath) throws IOException { + return Files.size(filePath) > properties.fileSizeLimit() * 1024L * 1024L; + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepository.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepository.java new file mode 100644 index 00000000000..fb62d16b933 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepository.java @@ -0,0 +1,34 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.scan.filesystem; + +import org.sonar.api.batch.fs.InputFileFilter; + +public class InputFileFilterRepository { + private final InputFileFilter[] inputFileFilters; + + public InputFileFilterRepository(InputFileFilter... inputFileFilters) { + this.inputFileFilters = inputFileFilters; + } + + public InputFileFilter[] getInputFileFilters() { + return inputFileFilters; + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java index bcd023e1ecd..3beaea8c255 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java @@ -25,6 +25,7 @@ import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.CheckForNull; @@ -53,8 +54,9 @@ public class LanguageDetection { */ private final Map<Language, PathPattern[]> patternsByLanguage; private final List<Language> languagesToConsider; + private final Map<String, Language> languageCacheByPath; - public LanguageDetection(Configuration settings, LanguagesRepository languages) { + public LanguageDetection(Configuration settings, LanguagesRepository languages, Map<String, Language> languageCache) { Map<Language, PathPattern[]> patternsByLanguageBuilder = new LinkedHashMap<>(); for (Language language : languages.all()) { String[] filePatterns = settings.getStringArray(getFileLangPatternPropKey(language.key())); @@ -69,6 +71,7 @@ public class LanguageDetection { languagesToConsider = List.copyOf(patternsByLanguageBuilder.keySet()); patternsByLanguage = unmodifiableMap(patternsByLanguageBuilder); + languageCacheByPath = languageCache; } private static PathPattern[] getLanguagePatterns(Language language) { @@ -89,11 +92,16 @@ public class LanguageDetection { @CheckForNull Language language(Path absolutePath, Path relativePath) { - Language detectedLanguage = null; + Language detectedLanguage = languageCacheByPath.get(absolutePath.toString()); + if (detectedLanguage != null) { + return detectedLanguage; + } + for (Language language : languagesToConsider) { if (isCandidateForLanguage(absolutePath, relativePath, language)) { if (detectedLanguage == null) { detectedLanguage = language; + languageCacheByPath.put(absolutePath.toString(), language); } else { // Language was already forced by another pattern throw MessageException.of(MessageFormat.format("Language of file ''{0}'' can not be decided as the file matches patterns of both {1} and {2}", @@ -105,6 +113,10 @@ public class LanguageDetection { return detectedLanguage; } + public Set<String> getDetectedLanguages() { + return languageCacheByPath.values().stream().map(Language::key).collect(Collectors.toSet()); + } + private boolean isCandidateForLanguage(Path absolutePath, Path relativePath, Language language) { PathPattern[] patterns = patternsByLanguage.get(language); return patterns != null && Arrays.stream(patterns).anyMatch(pattern -> pattern.match(absolutePath, relativePath, false)); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleRelativePathWarner.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleRelativePathWarner.java new file mode 100644 index 00000000000..c61cc59ed39 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleRelativePathWarner.java @@ -0,0 +1,47 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.scan.filesystem; + +import java.util.HashSet; +import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.notifications.AnalysisWarnings; + +public class ModuleRelativePathWarner { + + private static final Logger LOG = LoggerFactory.getLogger(ModuleRelativePathWarner.class); + private final AnalysisWarnings analysisWarnings; + private final Set<String> previouslyWarnedProps = new HashSet<>(); + + public ModuleRelativePathWarner(AnalysisWarnings analysisWarnings) { + this.analysisWarnings = analysisWarnings; + } + + public void warnOnce(String propKey, String filePath) { + if (!previouslyWarnedProps.contains(propKey)) { + previouslyWarnedProps.add(propKey); + String msg = "Specifying module-relative paths at project level in the property '" + propKey + "' is deprecated. " + + "To continue matching files like '" + filePath + "', update this property so that patterns refer to project-relative paths."; + LOG.warn(msg); + analysisWarnings.addUnique(msg); + } + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java index d7b66e616e9..e8c7086168a 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java @@ -20,33 +20,22 @@ package org.sonar.scanner.scan.filesystem; import java.io.IOException; -import java.nio.file.AccessDeniedException; -import java.nio.file.FileSystemLoopException; import java.nio.file.FileVisitOption; -import java.nio.file.FileVisitResult; -import java.nio.file.FileVisitor; import java.nio.file.Files; -import java.nio.file.LinkOption; import java.nio.file.Path; -import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.attribute.DosFileAttributes; import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.SystemUtils; -import org.sonar.api.batch.fs.InputFile; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.sonar.api.batch.fs.InputFile.Type; import org.sonar.api.batch.fs.internal.DefaultInputModule; -import org.sonar.api.batch.scm.IgnoreCommand; import org.sonar.api.notifications.AnalysisWarnings; import org.sonar.api.scan.filesystem.PathResolver; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.sonar.scanner.bootstrap.GlobalConfiguration; import org.sonar.scanner.bootstrap.GlobalServerSettings; import org.sonar.scanner.fs.InputModuleHierarchy; @@ -54,12 +43,8 @@ import org.sonar.scanner.scan.ModuleConfiguration; import org.sonar.scanner.scan.ModuleConfigurationProvider; import org.sonar.scanner.scan.ProjectServerSettings; import org.sonar.scanner.scan.SonarGlobalPropertiesFilter; -import org.sonar.scanner.scm.ScmConfiguration; import org.sonar.scanner.util.ProgressReport; -import static java.util.Collections.emptyList; -import static java.util.Collections.singletonList; - /** * Index project input files into {@link InputComponentStore}. */ @@ -69,15 +54,13 @@ public class ProjectFileIndexer { private final ProjectExclusionFilters projectExclusionFilters; private final SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter; private final ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions; - private final ScmConfiguration scmConfiguration; private final InputComponentStore componentStore; private final InputModuleHierarchy inputModuleHierarchy; private final GlobalConfiguration globalConfig; private final GlobalServerSettings globalServerSettings; private final ProjectServerSettings projectServerSettings; private final FileIndexer fileIndexer; - private final IgnoreCommand ignoreCommand; - private final boolean useScmExclusion; + private final ProjectFilePreprocessor projectFilePreprocessor; private final AnalysisWarnings analysisWarnings; private ProgressReport progressReport; @@ -85,8 +68,8 @@ public class ProjectFileIndexer { public ProjectFileIndexer(InputComponentStore componentStore, ProjectExclusionFilters exclusionFilters, SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter, InputModuleHierarchy inputModuleHierarchy, GlobalConfiguration globalConfig, GlobalServerSettings globalServerSettings, ProjectServerSettings projectServerSettings, - FileIndexer fileIndexer, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, ScmConfiguration scmConfiguration, - AnalysisWarnings analysisWarnings) { + FileIndexer fileIndexer, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, + ProjectFilePreprocessor projectFilePreprocessor, AnalysisWarnings analysisWarnings) { this.componentStore = componentStore; this.sonarGlobalPropertiesFilter = sonarGlobalPropertiesFilter; this.inputModuleHierarchy = inputModuleHierarchy; @@ -96,10 +79,8 @@ public class ProjectFileIndexer { this.fileIndexer = fileIndexer; this.projectExclusionFilters = exclusionFilters; this.projectCoverageAndDuplicationExclusions = projectCoverageAndDuplicationExclusions; - this.scmConfiguration = scmConfiguration; + this.projectFilePreprocessor = projectFilePreprocessor; this.analysisWarnings = analysisWarnings; - this.ignoreCommand = loadIgnoreCommand(); - this.useScmExclusion = ignoreCommand != null; } public void index() { @@ -108,47 +89,22 @@ public class ProjectFileIndexer { LOG.info("Project configuration:"); projectExclusionFilters.log(" "); projectCoverageAndDuplicationExclusions.log(" "); - ExclusionCounter exclusionCounter = new ExclusionCounter(); - if (useScmExclusion) { - ignoreCommand.init(inputModuleHierarchy.root().getBaseDir().toAbsolutePath()); - indexModulesRecursively(inputModuleHierarchy.root(), exclusionCounter); - ignoreCommand.clean(); - } else { - indexModulesRecursively(inputModuleHierarchy.root(), exclusionCounter); - } + indexModulesRecursively(inputModuleHierarchy.root()); int totalIndexed = componentStore.inputFiles().size(); progressReport.stop(totalIndexed + " " + pluralizeFiles(totalIndexed) + " indexed"); - int excludedFileByPatternCount = exclusionCounter.getByPatternsCount(); - if (projectExclusionFilters.hasPattern() || excludedFileByPatternCount > 0) { - LOG.info("{} {} ignored because of inclusion/exclusion patterns", excludedFileByPatternCount, pluralizeFiles(excludedFileByPatternCount)); - } - int excludedFileByScmCount = exclusionCounter.getByScmCount(); - if (useScmExclusion) { - LOG.info("{} {} ignored because of scm ignore settings", excludedFileByScmCount, pluralizeFiles(excludedFileByScmCount)); - } } - private IgnoreCommand loadIgnoreCommand() { - try { - if (!scmConfiguration.isExclusionDisabled() && scmConfiguration.provider() != null) { - return scmConfiguration.provider().ignoreCommand(); - } - } catch (UnsupportedOperationException e) { - LOG.debug("File exclusion based on SCM ignore information is not available with this plugin."); - } - - return null; + private void indexModulesRecursively(DefaultInputModule module) { + inputModuleHierarchy.children(module).stream() + .sorted(Comparator.comparing(DefaultInputModule::key)) + .forEach(this::indexModulesRecursively); + index(module); } - private void indexModulesRecursively(DefaultInputModule module, ExclusionCounter exclusionCounter) { - inputModuleHierarchy.children(module).stream().sorted(Comparator.comparing(DefaultInputModule::key)).forEach(m -> indexModulesRecursively(m, exclusionCounter)); - index(module, exclusionCounter); - } - - private void index(DefaultInputModule module, ExclusionCounter exclusionCounter) { + private void index(DefaultInputModule module) { // Emulate creation of module level settings ModuleConfiguration moduleConfig = new ModuleConfigurationProvider(sonarGlobalPropertiesFilter).provide(globalConfig, module, globalServerSettings, projectServerSettings); ModuleExclusionFilters moduleExclusionFilters = new ModuleExclusionFilters(moduleConfig, analysisWarnings); @@ -161,13 +117,10 @@ public class ProjectFileIndexer { moduleExclusionFilters.log(" "); moduleCoverageAndDuplicationExclusions.log(" "); } - boolean hasChildModules = !module.definition().getSubProjects().isEmpty(); - boolean hasTests = module.getTestDirsOrFiles().isPresent(); - // Default to index basedir when no sources provided - List<Path> mainSourceDirsOrFiles = module.getSourceDirsOrFiles() - .orElseGet(() -> hasChildModules || hasTests ? emptyList() : singletonList(module.getBaseDir().toAbsolutePath())); - indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, mainSourceDirsOrFiles, Type.MAIN, exclusionCounter); - module.getTestDirsOrFiles().ifPresent(tests -> indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, tests, Type.TEST, exclusionCounter)); + List<Path> mainSourceDirsOrFiles = projectFilePreprocessor.getMainSourcesByModule(module); + indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, mainSourceDirsOrFiles, Type.MAIN); + projectFilePreprocessor.getTestSourcesByModule(module) + .ifPresent(tests -> indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, tests, Type.TEST)); } private static void logPaths(String label, Path baseDir, List<Path> paths) { @@ -176,7 +129,7 @@ public class ProjectFileIndexer { for (Iterator<Path> it = paths.iterator(); it.hasNext(); ) { Path file = it.next(); Optional<String> relativePathToBaseDir = PathResolver.relativize(baseDir, file); - if (!relativePathToBaseDir.isPresent()) { + if (relativePathToBaseDir.isEmpty()) { sb.append(file); } else if (StringUtils.isBlank(relativePathToBaseDir.get())) { sb.append("."); @@ -195,19 +148,14 @@ public class ProjectFileIndexer { } } - private static String pluralizeFiles(int count) { - return count == 1 ? "file" : "files"; - } - - private void indexFiles(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, - ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, List<Path> sources, Type type, ExclusionCounter exclusionCounter) { + private void indexFiles(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, + List<Path> sources, Type type) { try { for (Path dirOrFile : sources) { if (dirOrFile.toFile().isDirectory()) { - indexDirectory(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type, exclusionCounter); + indexDirectory(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type); } else { - fileIndexer.indexFile(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type, progressReport, exclusionCounter, - ignoreCommand); + fileIndexer.indexFile(module, moduleCoverageAndDuplicationExclusions, dirOrFile, type, progressReport); } } } catch (IOException e) { @@ -216,141 +164,17 @@ public class ProjectFileIndexer { } private void indexDirectory(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, - ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path dirToIndex, Type type, ExclusionCounter exclusionCounter) - throws IOException { + ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, + Path dirToIndex, Type type) throws IOException { Files.walkFileTree(dirToIndex.normalize(), Collections.singleton(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, - new IndexFileVisitor(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, type, exclusionCounter)); - } - - - /** - * <p>Checks if the path is a directory that is excluded.</p> - * - * <p>Exclusions patterns are checked both at project and module level.</p> - * - * @param moduleExclusionFilters The exclusion filters. - * @param realAbsoluteFile The path to be checked. - * @param projectBaseDir The project base directory. - * @param moduleBaseDir The module base directory. - * @param type The input file type. - * @return True if path is an excluded directory, false otherwise. - */ - private static boolean isExcludedDirectory(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectBaseDir, Path moduleBaseDir, - InputFile.Type type) { - Path projectRelativePath = projectBaseDir.relativize(realAbsoluteFile); - Path moduleRelativePath = moduleBaseDir.relativize(realAbsoluteFile); - return moduleExclusionFilters.isExcludedAsParentDirectoryOfExcludedChildren(realAbsoluteFile, projectRelativePath, projectBaseDir, type) - || moduleExclusionFilters.isExcludedAsParentDirectoryOfExcludedChildren(realAbsoluteFile, moduleRelativePath, moduleBaseDir, type); + new DirectoryFileVisitor(file -> fileIndexer.indexFile(module, moduleCoverageAndDuplicationExclusions, file, type, progressReport), + module, moduleExclusionFilters, inputModuleHierarchy, type)); } - private class IndexFileVisitor implements FileVisitor<Path> { - private final DefaultInputModule module; - private final ModuleExclusionFilters moduleExclusionFilters; - private final ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions; - private final Type type; - private final ExclusionCounter exclusionCounter; - - IndexFileVisitor(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, - Type type, - ExclusionCounter exclusionCounter) { - this.module = module; - this.moduleExclusionFilters = moduleExclusionFilters; - this.moduleCoverageAndDuplicationExclusions = moduleCoverageAndDuplicationExclusions; - this.type = type; - this.exclusionCounter = exclusionCounter; - } - - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - if (isHidden(dir)) { - return FileVisitResult.SKIP_SUBTREE; - } - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - if (!Files.isHidden(file)) { - fileIndexer.indexFile(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, file, type, progressReport, exclusionCounter, ignoreCommand); - } - return FileVisitResult.CONTINUE; - } - - /** - * <p>Overridden method to handle exceptions while visiting files in the analysis.</p> - * - * <p> - * <ul> - * <li>FileSystemLoopException - We show a warning that a symlink loop exists and we skip the file.</li> - * <li>AccessDeniedException for excluded files/directories - We skip the file, as files excluded from the analysis, shouldn't throw access exceptions.</li> - * </ul> - * </p> - * - * @param file a reference to the file - * @param exc the I/O exception that prevented the file from being visited - * @throws IOException - */ - @Override - public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { - if (exc instanceof FileSystemLoopException) { - LOG.warn("Not indexing due to symlink loop: {}", file.toFile()); - return FileVisitResult.CONTINUE; - } else if (exc instanceof AccessDeniedException && isExcluded(file)) { - return FileVisitResult.CONTINUE; - } - throw exc; - } - - /** - * <p>Checks if the directory is excluded in the analysis or not. Only the exclusions are checked.</p> - * - * <p>The inclusions cannot be checked for directories, since the current implementation of pattern matching is intended only for files.</p> - * - * @param path The file or directory. - * @return True if file/directory is excluded from the analysis, false otherwise. - */ - private boolean isExcluded(Path path) throws IOException { - Path realAbsoluteFile = path.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize(); - return isExcludedDirectory(moduleExclusionFilters, realAbsoluteFile, inputModuleHierarchy.root().getBaseDir(), module.getBaseDir(), type); - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) { - return FileVisitResult.CONTINUE; - } - - private boolean isHidden(Path path) throws IOException { - if (SystemUtils.IS_OS_WINDOWS) { - try { - DosFileAttributes dosFileAttributes = Files.readAttributes(path, DosFileAttributes.class, LinkOption.NOFOLLOW_LINKS); - return dosFileAttributes.isHidden(); - } catch (UnsupportedOperationException e) { - return path.toFile().isHidden(); - } - } else { - return Files.isHidden(path); - } - } + private static String pluralizeFiles(int count) { + return count == 1 ? "file" : "files"; } - static class ExclusionCounter { - private final AtomicInteger excludedByPatternsCount = new AtomicInteger(0); - private final AtomicInteger excludedByScmCount = new AtomicInteger(0); - - public void increaseByPatternsCount() { - excludedByPatternsCount.incrementAndGet(); - } - - public int getByPatternsCount() { - return excludedByPatternsCount.get(); - } - public void increaseByScmCount() { - excludedByScmCount.incrementAndGet(); - } - public int getByScmCount() { - return excludedByScmCount.get(); - } - } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java new file mode 100644 index 00000000000..31a467fd5ce --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java @@ -0,0 +1,230 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.scan.filesystem; + +import java.io.IOException; +import java.nio.file.FileVisitOption; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.api.batch.scm.IgnoreCommand; +import org.sonar.api.batch.scm.ScmProvider; +import org.sonar.api.notifications.AnalysisWarnings; +import org.sonar.scanner.bootstrap.GlobalConfiguration; +import org.sonar.scanner.bootstrap.GlobalServerSettings; +import org.sonar.scanner.fs.InputModuleHierarchy; +import org.sonar.scanner.scan.ModuleConfiguration; +import org.sonar.scanner.scan.ModuleConfigurationProvider; +import org.sonar.scanner.scan.ProjectServerSettings; +import org.sonar.scanner.scan.SonarGlobalPropertiesFilter; +import org.sonar.scanner.scm.ScmConfiguration; +import org.sonar.scanner.util.ProgressReport; + +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; + +public class ProjectFilePreprocessor { + + private static final Logger LOG = LoggerFactory.getLogger(ProjectFilePreprocessor.class); + private static final String TELEMETRY_STEP_NAME = "file.preprocessing"; + + private final AnalysisWarnings analysisWarnings; + private final IgnoreCommand ignoreCommand; + private final boolean useScmExclusion; + private final ScmConfiguration scmConfiguration; + private final InputModuleHierarchy inputModuleHierarchy; + private final GlobalConfiguration globalConfig; + private final GlobalServerSettings globalServerSettings; + private final ProjectServerSettings projectServerSettings; + private final LanguageDetection languageDetection; + private final FilePreprocessor filePreprocessor; + private final ProjectExclusionFilters projectExclusionFilters; + + private final SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter; + + private final Map<DefaultInputModule, List<Path>> mainSourcesByModule = new HashMap<>(); + private final Map<DefaultInputModule, List<Path>> testSourcesByModule = new HashMap<>(); + + private int totalFilesPreprocessed = 0; + + public ProjectFilePreprocessor(AnalysisWarnings analysisWarnings, ScmConfiguration scmConfiguration, InputModuleHierarchy inputModuleHierarchy, + GlobalConfiguration globalConfig, GlobalServerSettings globalServerSettings, ProjectServerSettings projectServerSettings, + LanguageDetection languageDetection, FilePreprocessor filePreprocessor, + ProjectExclusionFilters projectExclusionFilters, SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter) { + this.analysisWarnings = analysisWarnings; + this.scmConfiguration = scmConfiguration; + this.inputModuleHierarchy = inputModuleHierarchy; + this.globalConfig = globalConfig; + this.globalServerSettings = globalServerSettings; + this.projectServerSettings = projectServerSettings; + this.languageDetection = languageDetection; + this.filePreprocessor = filePreprocessor; + this.projectExclusionFilters = projectExclusionFilters; + this.sonarGlobalPropertiesFilter = sonarGlobalPropertiesFilter; + this.ignoreCommand = loadIgnoreCommand(); + this.useScmExclusion = ignoreCommand != null; + } + + public void execute() { + ProgressReport progressReport = new ProgressReport("Report about progress of file preprocessing", + TimeUnit.SECONDS.toMillis(10)); + progressReport.start("Preprocessing files..."); + ExclusionCounter exclusionCounter = new ExclusionCounter(); + + if (useScmExclusion) { + ignoreCommand.init(inputModuleHierarchy.root().getBaseDir().toAbsolutePath()); + processModulesRecursively(inputModuleHierarchy.root(), exclusionCounter); + ignoreCommand.clean(); + } else { + processModulesRecursively(inputModuleHierarchy.root(), exclusionCounter); + } + + int totalLanguagesDetected = languageDetection.getDetectedLanguages().size(); + + progressReport.stop(String.format("%s detected in %s", pluralizeWithCount("language", totalLanguagesDetected), + pluralizeWithCount("preprocessed file", totalFilesPreprocessed))); + + int excludedFileByPatternCount = exclusionCounter.getByPatternsCount(); + if (projectExclusionFilters.hasPattern() || excludedFileByPatternCount > 0) { + if (LOG.isInfoEnabled()) { + LOG.info("{} ignored because of inclusion/exclusion patterns", pluralizeWithCount("file", excludedFileByPatternCount)); + } + } + + int excludedFileByScmCount = exclusionCounter.getByScmCount(); + if (useScmExclusion) { + if (LOG.isInfoEnabled()) { + LOG.info("{} ignored because of scm ignore settings", pluralizeWithCount("file", excludedFileByScmCount)); + } + } + } + + private void processModulesRecursively(DefaultInputModule module, ExclusionCounter exclusionCounter) { + inputModuleHierarchy.children(module).stream().sorted(Comparator.comparing(DefaultInputModule::key)).forEach( + m -> processModulesRecursively(m, exclusionCounter)); + processModule(module, exclusionCounter); + } + + private void processModule(DefaultInputModule module, ExclusionCounter exclusionCounter) { + // Emulate creation of module level settings + ModuleConfiguration moduleConfig = new ModuleConfigurationProvider(sonarGlobalPropertiesFilter).provide(globalConfig, module, globalServerSettings, projectServerSettings); + ModuleExclusionFilters moduleExclusionFilters = new ModuleExclusionFilters(moduleConfig, analysisWarnings); + boolean hasChildModules = !module.definition().getSubProjects().isEmpty(); + boolean hasTests = module.getTestDirsOrFiles().isPresent(); + // Default to index basedir when no sources provided + List<Path> mainSourceDirsOrFiles = module.getSourceDirsOrFiles() + .orElseGet(() -> hasChildModules || hasTests ? emptyList() : singletonList(module.getBaseDir().toAbsolutePath())); + List<Path> processedSources = processModuleSources(module, moduleExclusionFilters, mainSourceDirsOrFiles, InputFile.Type.MAIN, + exclusionCounter); + mainSourcesByModule.put(module, processedSources); + totalFilesPreprocessed += processedSources.size(); + module.getTestDirsOrFiles().ifPresent(tests -> { + List<Path> processedTestSources = processModuleSources(module, moduleExclusionFilters, tests, InputFile.Type.TEST, exclusionCounter); + testSourcesByModule.put(module, processedTestSources); + totalFilesPreprocessed += processedTestSources.size(); + }); + } + + private List<Path> processModuleSources(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, List<Path> sources, + InputFile.Type type, ExclusionCounter exclusionCounter) { + List<Path> processedFiles = new ArrayList<>(); + try { + for (Path dirOrFile : sources) { + if (dirOrFile.toFile().isDirectory()) { + processedFiles.addAll(processDirectory(module, moduleExclusionFilters, dirOrFile, type, exclusionCounter)); + } else { + filePreprocessor.processFile(module, moduleExclusionFilters, dirOrFile, type, exclusionCounter, ignoreCommand) + .ifPresent(processedFiles::add); + } + } + } catch (IOException e) { + throw new IllegalStateException("Failed to preprocess files", e); + } + return processedFiles; + } + + private List<Path> processDirectory(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, Path path, + InputFile.Type type, ExclusionCounter exclusionCounter) throws IOException { + List<Path> processedFiles = new ArrayList<>(); + Files.walkFileTree(path.normalize(), Collections.singleton(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, + new DirectoryFileVisitor(file -> filePreprocessor.processFile(module, moduleExclusionFilters, file, type, exclusionCounter, + ignoreCommand).ifPresent(processedFiles::add), module, moduleExclusionFilters, inputModuleHierarchy, type) + ); + return processedFiles; + } + + public List<Path> getMainSourcesByModule(DefaultInputModule module) { + return Collections.unmodifiableList(mainSourcesByModule.get(module)); + } + + public Optional<List<Path>> getTestSourcesByModule(DefaultInputModule module) { + return Optional.ofNullable(testSourcesByModule.get(module)).map(Collections::unmodifiableList); + } + + private IgnoreCommand loadIgnoreCommand() { + try { + ScmProvider provider = scmConfiguration.provider(); + if (!scmConfiguration.isExclusionDisabled() && provider != null) { + return provider.ignoreCommand(); + } + } catch (UnsupportedOperationException e) { + LOG.debug("File exclusion based on SCM ignore information is not available with this plugin."); + } + + return null; + } + + private static String pluralizeWithCount(String str, int count) { + String pluralized = count == 1 ? str : (str + "s"); + return count + " " + pluralized; + } + + public static class ExclusionCounter { + private int excludedByPatternsCount = 0; + private int excludedByScmCount = 0; + + public void increaseByPatternsCount() { + excludedByPatternsCount++; + } + + public int getByPatternsCount() { + return excludedByPatternsCount; + } + + public void increaseByScmCount() { + excludedByScmCount++; + } + + public int getByScmCount() { + return excludedByScmCount; + } + } +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitorTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitorTest.java new file mode 100644 index 00000000000..b7794c959a8 --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitorTest.java @@ -0,0 +1,93 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.scan.filesystem; + +import java.io.File; +import java.io.IOException; +import java.nio.file.FileSystemLoopException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import org.apache.commons.lang.SystemUtils; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.scanner.fs.InputModuleHierarchy; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +public class DirectoryFileVisitorTest { + + @ClassRule + public static TemporaryFolder temp = new TemporaryFolder(); + + private final DefaultInputModule module = mock(); + private final ModuleExclusionFilters moduleExclusionFilters = mock(); + private final InputModuleHierarchy inputModuleHierarchy = mock(); + private final InputFile.Type type = mock(); + + @Test + public void visit_hidden_file() throws IOException { + DirectoryFileVisitor.FileVisitAction action = mock(DirectoryFileVisitor.FileVisitAction.class); + + File hidden = temp.newFile(".hidden"); + if (SystemUtils.IS_OS_WINDOWS) { + Files.setAttribute(hidden.toPath(), "dos:hidden", true, LinkOption.NOFOLLOW_LINKS); + } + + + DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleExclusionFilters, inputModuleHierarchy, type); + underTest.visitFile(hidden.toPath(), Files.readAttributes(hidden.toPath(), BasicFileAttributes.class)); + + verify(action, never()).execute(any(Path.class)); + } + + @Test + public void test_visit_file_failed_generic_io_exception() throws IOException { + DirectoryFileVisitor.FileVisitAction action = mock(DirectoryFileVisitor.FileVisitAction.class); + + File file = temp.newFile("failed"); + + DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleExclusionFilters, inputModuleHierarchy, type); + assertThrows(IOException.class, () -> underTest.visitFileFailed(file.toPath(), new IOException())); + } + + @Test + public void test_visit_file_failed_file_system_loop_exception() throws IOException { + DirectoryFileVisitor.FileVisitAction action = mock(DirectoryFileVisitor.FileVisitAction.class); + + File file = temp.newFile("symlink"); + + DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleExclusionFilters, inputModuleHierarchy, type); + FileVisitResult result = underTest.visitFileFailed(file.toPath(), new FileSystemLoopException(file.getPath())); + + assertThat(result).isEqualTo(FileVisitResult.CONTINUE); + } + +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepositoryTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepositoryTest.java new file mode 100644 index 00000000000..7be01e0a79f --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepositoryTest.java @@ -0,0 +1,41 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.scan.filesystem; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class InputFileFilterRepositoryTest { + + @Test + public void should_not_return_null_if_initialized_with_no_filters() { + InputFileFilterRepository underTest = new InputFileFilterRepository(); + assertThat(underTest.getInputFileFilters()).isNotNull(); + } + + @Test + public void should_return_filters_from_initialization() { + InputFileFilterRepository underTest = new InputFileFilterRepository(f -> true); + assertThat(underTest.getInputFileFilters()).isNotNull(); + assertThat(underTest.getInputFileFilters()).hasSize(1); + } + +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java index 34637645cbc..1440d8f427b 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java @@ -24,6 +24,8 @@ import com.tngtech.java.junit.dataprovider.DataProviderRunner; import com.tngtech.java.junit.dataprovider.UseDataProvider; import java.io.File; import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -38,7 +40,11 @@ import org.sonar.scanner.repository.language.LanguagesRepository; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.endsWith; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; @RunWith(DataProviderRunner.class) public class LanguageDetectionTest { @@ -72,7 +78,7 @@ public class LanguageDetectionTest { @Test public void detectLanguageKey_shouldDetectByFileExtension() { LanguagesRepository languages = new FakeLanguagesRepository(new Languages(new MockLanguage("java", "java", "jav"), new MockLanguage("cobol", "cbl", "cob"))); - LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages); + LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>()); assertThat(detectLanguageKey(detection, "Foo.java")).isEqualTo("java"); assertThat(detectLanguageKey(detection, "src/Foo.java")).isEqualTo("java"); @@ -94,7 +100,7 @@ public class LanguageDetectionTest { new MockLanguage("docker", new String[0], new String[] {"*.dockerfile", "*.Dockerfile", "Dockerfile", "Dockerfile.*"}), new MockLanguage("terraform", new String[] {"tf"}, new String[] {".tf"}), new MockLanguage("java", new String[0], new String[] {"**/*Test.java"}))); - LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages); + LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>()); assertThat(detectLanguageKey(detection, fileName)).isEqualTo(expectedLanguageKey); } @@ -117,7 +123,7 @@ public class LanguageDetectionTest { @Test public void detectLanguageKey_shouldNotFailIfNoLanguage() { - LanguageDetection detection = spy(new LanguageDetection(settings.asConfig(), new FakeLanguagesRepository(new Languages()))); + LanguageDetection detection = spy(new LanguageDetection(settings.asConfig(), new FakeLanguagesRepository(new Languages()), new HashMap<>())); assertThat(detectLanguageKey(detection, "Foo.java")).isNull(); } @@ -125,14 +131,14 @@ public class LanguageDetectionTest { public void detectLanguageKey_shouldAllowPluginsToDeclareFileExtensionTwiceForCaseSensitivity() { LanguagesRepository languages = new FakeLanguagesRepository(new Languages(new MockLanguage("abap", "abap", "ABAP"))); - LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages); + LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>()); assertThat(detectLanguageKey(detection, "abc.abap")).isEqualTo("abap"); } @Test public void detectLanguageKey_shouldFailIfConflictingLanguageSuffix() { LanguagesRepository languages = new FakeLanguagesRepository(new Languages(new MockLanguage("xml", "xhtml"), new MockLanguage("web", "xhtml"))); - LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages); + LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>()); assertThatThrownBy(() -> detectLanguageKey(detection, "abc.xhtml")) .isInstanceOf(MessageException.class) .hasMessageContaining("Language of file 'abc.xhtml' can not be decided as the file matches patterns of both ") @@ -146,7 +152,7 @@ public class LanguageDetectionTest { settings.setProperty("sonar.lang.patterns.xml", "xml/**"); settings.setProperty("sonar.lang.patterns.web", "web/**"); - LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages); + LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>()); assertThat(detectLanguageKey(detection, "xml/abc.xhtml")).isEqualTo("xml"); assertThat(detectLanguageKey(detection, "web/abc.xhtml")).isEqualTo("web"); } @@ -157,7 +163,7 @@ public class LanguageDetectionTest { settings.setProperty("sonar.lang.patterns.abap", "*.abap,*.txt"); settings.setProperty("sonar.lang.patterns.cobol", "*.cobol,*.txt"); - LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages); + LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>()); assertThat(detectLanguageKey(detection, "abc.abap")).isEqualTo("abap"); assertThat(detectLanguageKey(detection, "abc.cobol")).isEqualTo("cobol"); @@ -168,6 +174,19 @@ public class LanguageDetectionTest { .hasMessageContaining("sonar.lang.patterns.cobol : *.cobol,*.txt"); } + @Test + public void should_cache_detected_language_by_file_path() { + Map<String, org.sonar.scanner.repository.language.Language> languageCacheSpy = spy(new HashMap<>()); + LanguagesRepository languages = new FakeLanguagesRepository(new Languages( + new MockLanguage("java", "java", "jav"), new MockLanguage("cobol", "cbl", "cob"))); + LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, languageCacheSpy); + + assertThat(detectLanguageKey(detection, "Foo.java")).isEqualTo("java"); + assertThat(detectLanguageKey(detection, "Foo.java")).isEqualTo("java"); + verify(languageCacheSpy, times(1)).put(endsWith("/Foo.java"), any(org.sonar.scanner.repository.language.Language.class)); + verify(languageCacheSpy, times(2)).get(endsWith("/Foo.java")); + } + private String detectLanguageKey(LanguageDetection detection, String path) { org.sonar.scanner.repository.language.Language language = detection.language(new File(temp.getRoot(), path).toPath(), Paths.get(path)); return language != null ? language.key() : null; diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/sonar-project.properties b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/sonar-project.properties new file mode 100644 index 00000000000..57c2c062b0e --- /dev/null +++ b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/sonar-project.properties @@ -0,0 +1,5 @@ +sonar.organization=org1 +sonar.projectKey=sample-with-empty-file +sonar.projectName=Sample With Empty +sonar.projectVersion=0.1-SNAPSHOT +sonar.sources=xources diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/HelloJava.xoo b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/HelloJava.xoo new file mode 100644 index 00000000000..ee9bf789a53 --- /dev/null +++ b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/HelloJava.xoo @@ -0,0 +1,8 @@ +package hello; + +public class HelloJava { + + public static void main(String[] args) { + System.out.println("Hello"); + } +} diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude.xoo b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude.xoo new file mode 100644 index 00000000000..35965a8484c --- /dev/null +++ b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude.xoo @@ -0,0 +1 @@ +this file should be excluded from indexing. diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude2.xoo b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude2.xoo new file mode 100644 index 00000000000..d04e465a561 --- /dev/null +++ b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude2.xoo @@ -0,0 +1 @@ +this file should ALSO be excluded from indexing. |