aboutsummaryrefslogtreecommitdiffstats
path: root/sonar-scanner-engine
diff options
context:
space:
mode:
authorMatteo Mara <matteo.mara@sonarsource.com>2023-12-22 10:28:20 +0100
committersonartech <sonartech@sonarsource.com>2023-12-22 20:03:01 +0000
commit48705fd9366d3cfae6c4642084d0400ee73313ad (patch)
tree1932ef20c1ff00118d8b83a6879e16d62a788381 /sonar-scanner-engine
parentb6367ef978efd8f342dd70c4427e7570320818a4 (diff)
downloadsonarqube-48705fd9366d3cfae6c4642084d0400ee73313ad.tar.gz
sonarqube-48705fd9366d3cfae6c4642084d0400ee73313ad.zip
Revert "SONAR-21195 Refactor file indexing into two distinct steps"
This reverts commit 42d16dafde01b19bc7fcb99084e087fdc9b8e95e.
Diffstat (limited to 'sonar-scanner-engine')
-rw-r--r--sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/ScannerMediumTester.java8
-rw-r--r--sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java43
-rw-r--r--sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java11
-rw-r--r--sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java11
-rw-r--r--sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java20
-rw-r--r--sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java12
-rw-r--r--sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java21
-rw-r--r--sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/LocalPlugin.java33
-rw-r--r--sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java15
-rw-r--r--sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java152
-rw-r--r--sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java141
-rw-r--r--sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java139
-rw-r--r--sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepository.java34
-rw-r--r--sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java16
-rw-r--r--sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleRelativePathWarner.java47
-rw-r--r--sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java232
-rw-r--r--sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java230
-rw-r--r--sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitorTest.java93
-rw-r--r--sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepositoryTest.java41
-rw-r--r--sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java33
-rw-r--r--sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/sonar-project.properties5
-rw-r--r--sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/HelloJava.xoo8
-rw-r--r--sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude.xoo1
-rw-r--r--sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude2.xoo1
24 files changed, 355 insertions, 992 deletions
diff --git a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/ScannerMediumTester.java b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/ScannerMediumTester.java
index 1916d3e6d00..ebf0dc20efa 100644
--- a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/ScannerMediumTester.java
+++ b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/ScannerMediumTester.java
@@ -35,7 +35,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
-import java.util.Set;
import javax.annotation.CheckForNull;
import javax.annotation.Nullable;
import javax.annotation.Priority;
@@ -137,12 +136,11 @@ public class ScannerMediumTester extends ExternalResource {
}
public ScannerMediumTester registerPlugin(String pluginKey, Plugin instance) {
- pluginInstaller.add(pluginKey, instance);
- return this;
+ return registerPlugin(pluginKey, instance, 1L);
}
- public ScannerMediumTester registerOptionalPlugin(String pluginKey, Set<String> requiredForLanguages, Plugin instance) {
- pluginInstaller.addOptional(pluginKey, requiredForLanguages, instance);
+ public ScannerMediumTester registerPlugin(String pluginKey, Plugin instance, long lastUpdatedAt) {
+ pluginInstaller.add(pluginKey, instance, lastUpdatedAt);
return this;
}
diff --git a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java
index 66f35f9cfa8..0bfef5a38ad 100644
--- a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java
+++ b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java
@@ -29,7 +29,6 @@ import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Random;
-import java.util.Set;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.SystemUtils;
@@ -40,10 +39,8 @@ import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.slf4j.event.Level;
import org.sonar.api.CoreProperties;
-import org.sonar.api.Plugin;
import org.sonar.api.SonarEdition;
import org.sonar.api.batch.fs.InputFile;
-import org.sonar.api.batch.fs.InputFileFilter;
import org.sonar.api.batch.fs.internal.DefaultInputFile;
import org.sonar.api.testfixtures.log.LogTester;
import org.sonar.api.utils.MessageException;
@@ -77,7 +74,6 @@ public class FileSystemMediumIT {
public ScannerMediumTester tester = new ScannerMediumTester()
.setEdition(SonarEdition.COMMUNITY)
.registerPlugin("xoo", new XooPlugin())
- .registerOptionalPlugin("optional-xoo", Set.of("xoo"), new OptionalXooPlugin())
.addDefaultQProfile("xoo", "Sonar Way")
.addDefaultQProfile("xoo2", "Sonar Way");
@@ -1185,7 +1181,7 @@ public class FileSystemMediumIT {
assertThatThrownBy(result::execute)
.isExactlyInstanceOf(IllegalStateException.class)
- .hasMessageEndingWith(format("Failed to preprocess files"));
+ .hasMessageEndingWith(format("Failed to index files"));
}
@Test
@@ -1256,42 +1252,7 @@ public class FileSystemMediumIT {
assertThatThrownBy(result::execute)
.isExactlyInstanceOf(IllegalStateException.class)
- .hasMessageEndingWith(format("Failed to preprocess files"));
- }
-
- @Test
- public void should_load_input_file_filters_for_required_and_optional_plugins() throws IOException {
- File projectDir = new File("test-resources/mediumtest/xoo/sample-with-input-file-filters");
- AnalysisResult result = tester
- .newAnalysis(new File(projectDir, "sonar-project.properties"))
- .execute();
-
- assertThat(result.inputFiles()).hasSize(1);
-
- assertThat(logTester.logs()).contains("'xources/hello/xoo_exclude2.xoo' excluded by org.sonar.scanner.mediumtest.fs" +
- ".FileSystemMediumIT$OptionalXooPlugin$OptionalXooFileFilter");
- assertThat(logTester.logs()).contains("'xources/hello/xoo_exclude.xoo' excluded by org.sonar.xoo.extensions.XooExcludeFileFilter");
- assertThat(logTester.logs()).contains("'xources/hello/HelloJava.xoo' indexed with language 'xoo'");
-
- assertThat(result.inputFile("xources/hello/xoo_exclude.xoo")).isNull();
- assertThat(result.inputFile("xources/hello/xoo_exclude2.xoo")).isNull();
- assertThat(result.inputFile("xources/hello/HelloJava.xoo")).isNotNull();
- }
-
- public static class OptionalXooPlugin implements Plugin {
-
- @Override
- public void define(Context context) {
- context.addExtension(OptionalXooFileFilter.class);
- }
-
- public static class OptionalXooFileFilter implements InputFileFilter {
-
- @Override
- public boolean accept(InputFile f) {
- return !f.filename().endsWith("_exclude2.xoo");
- }
- }
+ .hasMessageEndingWith(format("Failed to index files"));
}
private static void assertAnalysedFiles(AnalysisResult result, String... files) {
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java
index de69726669c..eeb35462dab 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java
@@ -22,7 +22,6 @@ package org.sonar.scanner.bootstrap;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.sonar.scanner.mediumtest.LocalPlugin;
public interface PluginInstaller {
@@ -46,14 +45,8 @@ public interface PluginInstaller {
Map<String, ScannerPlugin> installPluginsForLanguages(Set<String> languageKeys);
/**
- * Used only by medium tests. Installs required plugins (phase 1)
+ * Used only by medium tests.
* @see org.sonar.scanner.mediumtest.ScannerMediumTester
*/
- List<LocalPlugin> installLocals();
-
- /**
- * Used only by medium tests. Installs optional plugins (phase 2)
- * @see org.sonar.scanner.mediumtest.ScannerMediumTester
- */
- List<LocalPlugin> installOptionalLocals(Set<String> languageKeys);
+ List<Object[]> installLocals();
}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java
index e05b7d08471..efed3403147 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java
@@ -36,7 +36,6 @@ import org.sonar.api.utils.log.Loggers;
import org.sonar.api.utils.log.Profiler;
import org.sonar.core.platform.PluginInfo;
import org.sonar.core.plugin.PluginType;
-import org.sonar.scanner.mediumtest.LocalPlugin;
import org.sonarqube.ws.client.GetRequest;
import static java.lang.String.format;
@@ -138,15 +137,7 @@ public class ScannerPluginInstaller implements PluginInstaller {
* Returns empty on purpose. This method is used only by medium tests.
*/
@Override
- public List<LocalPlugin> installLocals() {
- return Collections.emptyList();
- }
-
- /**
- * Returns empty on purpose. This method is used only by medium tests.
- */
- @Override
- public List<LocalPlugin> installOptionalLocals(Set<String> languageKeys) {
+ public List<Object[]> installLocals() {
return Collections.emptyList();
}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java
index 074079f974e..3ce0202292b 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java
@@ -36,7 +36,6 @@ import org.sonar.core.platform.PluginInfo;
import org.sonar.core.platform.PluginJarExploder;
import org.sonar.core.platform.PluginRepository;
import org.sonar.core.plugin.PluginType;
-import org.sonar.scanner.mediumtest.LocalPlugin;
import static java.util.stream.Collectors.toMap;
import static org.sonar.api.utils.Preconditions.checkState;
@@ -80,11 +79,11 @@ public class ScannerPluginRepository implements PluginRepository, Startable {
pluginInstancesByKeys = new HashMap<>(loader.load(explodedPluginsByKey));
// this part is only used by medium tests
- for (LocalPlugin localPlugin : installer.installLocals()) {
- ScannerPlugin scannerPlugin = localPlugin.toScannerPlugin();
- String pluginKey = localPlugin.pluginKey();
- pluginsByKeys.put(pluginKey, scannerPlugin);
- pluginInstancesByKeys.put(pluginKey, localPlugin.pluginInstance());
+ for (Object[] localPlugin : installer.installLocals()) {
+ String pluginKey = (String) localPlugin[0];
+ PluginInfo pluginInfo = new PluginInfo(pluginKey);
+ pluginsByKeys.put(pluginKey, new ScannerPlugin(pluginInfo.getKey(), (long) localPlugin[2], PluginType.BUNDLED, pluginInfo));
+ pluginInstancesByKeys.put(pluginKey, (Plugin) localPlugin[1]);
}
keysByClassLoader = new HashMap<>();
@@ -108,15 +107,6 @@ public class ScannerPluginRepository implements PluginRepository, Startable {
.collect(toMap(Map.Entry::getKey, e -> pluginJarExploder.explode(e.getValue().getInfo())));
pluginInstancesByKeys.putAll(new HashMap<>(loader.load(explodedPluginsByKey)));
- // this part is only used by medium tests
- for (LocalPlugin localPlugin : installer.installOptionalLocals(languageKeys)) {
- ScannerPlugin scannerPlugin = localPlugin.toScannerPlugin();
- String pluginKey = localPlugin.pluginKey();
- languagePluginsByKeys.put(pluginKey, scannerPlugin);
- pluginsByKeys.put(pluginKey, scannerPlugin);
- pluginInstancesByKeys.put(pluginKey, localPlugin.pluginInstance());
- }
-
keysByClassLoader = new HashMap<>();
for (Map.Entry<String, Plugin> e : pluginInstancesByKeys.entrySet()) {
keysByClassLoader.put(e.getValue().getClass().getClassLoader(), e.getKey());
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java
index 38972c95071..eedc07f9924 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java
@@ -112,14 +112,13 @@ import org.sonar.scanner.scan.branch.BranchConfigurationProvider;
import org.sonar.scanner.scan.branch.BranchType;
import org.sonar.scanner.scan.branch.ProjectBranchesProvider;
import org.sonar.scanner.scan.filesystem.DefaultProjectFileSystem;
-import org.sonar.scanner.scan.filesystem.FilePreprocessor;
+import org.sonar.scanner.scan.filesystem.FileIndexer;
import org.sonar.scanner.scan.filesystem.InputComponentStore;
import org.sonar.scanner.scan.filesystem.LanguageDetection;
import org.sonar.scanner.scan.filesystem.MetadataGenerator;
-import org.sonar.scanner.scan.filesystem.ModuleRelativePathWarner;
import org.sonar.scanner.scan.filesystem.ProjectCoverageAndDuplicationExclusions;
import org.sonar.scanner.scan.filesystem.ProjectExclusionFilters;
-import org.sonar.scanner.scan.filesystem.ProjectFilePreprocessor;
+import org.sonar.scanner.scan.filesystem.ProjectFileIndexer;
import org.sonar.scanner.scan.filesystem.ScannerComponentIdGenerator;
import org.sonar.scanner.scan.filesystem.StatusDetection;
import org.sonar.scanner.scan.measure.DefaultMetricFinder;
@@ -194,9 +193,8 @@ public class SpringScannerContainer extends SpringComponentContainer {
LanguageDetection.class,
MetadataGenerator.class,
FileMetadata.class,
- ModuleRelativePathWarner.class,
- FilePreprocessor.class,
- ProjectFilePreprocessor.class,
+ FileIndexer.class,
+ ProjectFileIndexer.class,
ProjectExclusionFilters.class,
// rules
@@ -337,7 +335,7 @@ public class SpringScannerContainer extends SpringComponentContainer {
getComponentByType(DeprecatedPropertiesWarningGenerator.class).execute();
- getComponentByType(ProjectFilePreprocessor.class).execute();
+ getComponentByType(ProjectFileIndexer.class).index();
new SpringProjectScanContainer(this).execute();
}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java
index 5aaabab3980..fb48e11b424 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java
@@ -36,21 +36,15 @@ import org.sonar.scanner.bootstrap.ScannerPlugin;
public class FakePluginInstaller implements PluginInstaller {
private final Map<String, ScannerPlugin> pluginsByKeys = new HashMap<>();
- private final List<LocalPlugin> mediumTestPlugins = new ArrayList<>();
- private final List<LocalPlugin> optionalMediumTestPlugins = new ArrayList<>();
+ private final List<Object[]> mediumTestPlugins = new ArrayList<>();
public FakePluginInstaller add(String pluginKey, File jarFile, long lastUpdatedAt) {
pluginsByKeys.put(pluginKey, new ScannerPlugin(pluginKey, lastUpdatedAt, PluginType.BUNDLED, PluginInfo.create(jarFile)));
return this;
}
- public FakePluginInstaller add(String pluginKey, Plugin instance) {
- mediumTestPlugins.add(new LocalPlugin(pluginKey, instance, Set.of()));
- return this;
- }
-
- public FakePluginInstaller addOptional(String pluginKey, Set<String> requiredForLanguages, Plugin instance) {
- optionalMediumTestPlugins.add(new LocalPlugin(pluginKey, instance, requiredForLanguages));
+ public FakePluginInstaller add(String pluginKey, Plugin instance, long lastUpdatedAt) {
+ mediumTestPlugins.add(new Object[] {pluginKey, instance, lastUpdatedAt});
return this;
}
@@ -70,14 +64,7 @@ public class FakePluginInstaller implements PluginInstaller {
}
@Override
- public List<LocalPlugin> installLocals() {
+ public List<Object[]> installLocals() {
return mediumTestPlugins;
}
-
- @Override
- public List<LocalPlugin> installOptionalLocals(Set<String> languageKeys) {
- return optionalMediumTestPlugins.stream()
- .filter(plugin -> languageKeys.stream().anyMatch(lang -> plugin.requiredForLanguages().contains(lang)))
- .toList();
- }
}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/LocalPlugin.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/LocalPlugin.java
deleted file mode 100644
index 20bd4895973..00000000000
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/LocalPlugin.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2023 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.scanner.mediumtest;
-
-import java.util.Set;
-import org.sonar.api.Plugin;
-import org.sonar.core.platform.PluginInfo;
-import org.sonar.core.plugin.PluginType;
-import org.sonar.scanner.bootstrap.ScannerPlugin;
-
-public record LocalPlugin(String pluginKey, Plugin pluginInstance, Set<String> requiredForLanguages) {
-
- public ScannerPlugin toScannerPlugin() {
- return new ScannerPlugin(pluginKey, 1L, PluginType.BUNDLED, new PluginInfo(pluginKey));
- }
-}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java
index b70fde51660..351593e9404 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java
@@ -49,10 +49,7 @@ import org.sonar.scanner.qualitygate.QualityGateCheck;
import org.sonar.scanner.report.ContextPropertiesPublisher;
import org.sonar.scanner.report.ReportPublisher;
import org.sonar.scanner.rule.QProfileVerifier;
-import org.sonar.scanner.scan.filesystem.FileIndexer;
-import org.sonar.scanner.scan.filesystem.InputFileFilterRepository;
-import org.sonar.scanner.scan.filesystem.LanguageDetection;
-import org.sonar.scanner.scan.filesystem.ProjectFileIndexer;
+import org.sonar.scanner.scan.filesystem.InputComponentStore;
import org.sonar.scanner.scm.ScmPublisher;
import org.sonar.scanner.sensor.ProjectSensorExtensionDictionary;
import org.sonar.scanner.sensor.ProjectSensorsExecutor;
@@ -72,7 +69,7 @@ public class SpringProjectScanContainer extends SpringComponentContainer {
@Override
protected void doBeforeStart() {
- Set<String> languages = getParentComponentByType(LanguageDetection.class).getDetectedLanguages();
+ Set<String> languages = getParentComponentByType(InputComponentStore.class).languages();
installPluginsForLanguages(languages);
addScannerComponents();
}
@@ -117,12 +114,7 @@ public class SpringProjectScanContainer extends SpringComponentContainer {
ProjectSensorExtensionDictionary.class,
ProjectSensorsExecutor.class,
- AnalysisObservers.class,
-
- // file system
- InputFileFilterRepository.class,
- FileIndexer.class,
- ProjectFileIndexer.class);
+ AnalysisObservers.class);
}
static ExtensionMatcher getScannerProjectExtensionsFilter() {
@@ -138,7 +130,6 @@ public class SpringProjectScanContainer extends SpringComponentContainer {
protected void doAfterStart() {
getParentComponentByType(ScannerMetrics.class).addPluginMetrics(getComponentsByType(Metrics.class));
getComponentByType(ProjectLock.class).tryLock();
- getComponentByType(ProjectFileIndexer.class).index();
GlobalAnalysisMode analysisMode = getComponentByType(GlobalAnalysisMode.class);
InputModuleHierarchy tree = getComponentByType(InputModuleHierarchy.class);
ScanProperties properties = getComponentByType(ScanProperties.class);
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java
deleted file mode 100644
index eec0ed9c8d0..00000000000
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2023 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.scanner.scan.filesystem;
-
-import java.io.IOException;
-import java.nio.file.AccessDeniedException;
-import java.nio.file.FileSystemLoopException;
-import java.nio.file.FileVisitResult;
-import java.nio.file.FileVisitor;
-import java.nio.file.Files;
-import java.nio.file.LinkOption;
-import java.nio.file.Path;
-import java.nio.file.attribute.BasicFileAttributes;
-import java.nio.file.attribute.DosFileAttributes;
-import org.apache.commons.lang.SystemUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.sonar.api.batch.fs.InputFile;
-import org.sonar.api.batch.fs.internal.DefaultInputModule;
-import org.sonar.scanner.fs.InputModuleHierarchy;
-
-public class DirectoryFileVisitor implements FileVisitor<Path> {
-
- private static final Logger LOG = LoggerFactory.getLogger(DirectoryFileVisitor.class);
-
- private final FileVisitAction fileVisitAction;
- private final DefaultInputModule module;
- private final ModuleExclusionFilters moduleExclusionFilters;
-
- private final InputModuleHierarchy inputModuleHierarchy;
- private final InputFile.Type type;
-
- DirectoryFileVisitor(FileVisitAction fileVisitAction, DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, InputModuleHierarchy inputModuleHierarchy, InputFile.Type type) {
- this.fileVisitAction = fileVisitAction;
- this.module = module;
- this.moduleExclusionFilters = moduleExclusionFilters;
- this.inputModuleHierarchy = inputModuleHierarchy;
- this.type = type;
- }
-
- @Override
- public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
- return isHidden(dir) ? FileVisitResult.SKIP_SUBTREE : FileVisitResult.CONTINUE;
- }
-
- @Override
- public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
- if (!Files.isHidden(file)) {
- fileVisitAction.execute(file);
- }
- return FileVisitResult.CONTINUE;
- }
-
- /**
- * <p>Overridden method to handle exceptions while visiting files in the analysis.</p>
- *
- * <p>
- * <ul>
- * <li>FileSystemLoopException - We show a warning that a symlink loop exists and we skip the file.</li>
- * <li>AccessDeniedException for excluded files/directories - We skip the file, as files excluded from the analysis, shouldn't throw access exceptions.</li>
- * </ul>
- * </p>
- *
- * @param file a reference to the file
- * @param exc the I/O exception that prevented the file from being visited
- * @throws IOException
- */
- @Override
- public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
- if (exc instanceof FileSystemLoopException) {
- LOG.warn("Not indexing due to symlink loop: {}", file.toFile());
- return FileVisitResult.CONTINUE;
- } else if (exc instanceof AccessDeniedException && isExcluded(file)) {
- return FileVisitResult.CONTINUE;
- }
- throw exc;
- }
-
- /**
- * <p>Checks if the directory is excluded in the analysis or not. Only the exclusions are checked.</p>
- *
- * <p>The inclusions cannot be checked for directories, since the current implementation of pattern matching is intended only for files.</p>
- *
- * @param path The file or directory.
- * @return True if file/directory is excluded from the analysis, false otherwise.
- */
- private boolean isExcluded(Path path) throws IOException {
- Path realAbsoluteFile = path.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize();
- return isExcludedDirectory(moduleExclusionFilters, realAbsoluteFile, inputModuleHierarchy.root().getBaseDir(), module.getBaseDir(), type);
- }
-
- /**
- * <p>Checks if the path is a directory that is excluded.</p>
- *
- * <p>Exclusions patterns are checked both at project and module level.</p>
- *
- * @param moduleExclusionFilters The exclusion filters.
- * @param realAbsoluteFile The path to be checked.
- * @param projectBaseDir The project base directory.
- * @param moduleBaseDir The module base directory.
- * @param type The input file type.
- * @return True if path is an excluded directory, false otherwise.
- */
- private static boolean isExcludedDirectory(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectBaseDir, Path moduleBaseDir,
- InputFile.Type type) {
- Path projectRelativePath = projectBaseDir.relativize(realAbsoluteFile);
- Path moduleRelativePath = moduleBaseDir.relativize(realAbsoluteFile);
- return moduleExclusionFilters.isExcludedAsParentDirectoryOfExcludedChildren(realAbsoluteFile, projectRelativePath, projectBaseDir, type)
- || moduleExclusionFilters.isExcludedAsParentDirectoryOfExcludedChildren(realAbsoluteFile, moduleRelativePath, moduleBaseDir, type);
- }
-
- @Override
- public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
- return FileVisitResult.CONTINUE;
- }
-
- private static boolean isHidden(Path path) throws IOException {
- if (SystemUtils.IS_OS_WINDOWS) {
- try {
- DosFileAttributes dosFileAttributes = Files.readAttributes(path, DosFileAttributes.class, LinkOption.NOFOLLOW_LINKS);
- return dosFileAttributes.isHidden();
- } catch (UnsupportedOperationException e) {
- return path.toFile().isHidden();
- }
- } else {
- return Files.isHidden(path);
- }
- }
-
- @FunctionalInterface
- interface FileVisitAction {
- void execute(Path file) throws IOException;
- }
-}
-
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java
index 4d264590716..4d9b8c2a377 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java
@@ -19,10 +19,14 @@
*/
package org.sonar.scanner.scan.filesystem;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.util.function.BooleanSupplier;
+import javax.annotation.Nullable;
+import org.apache.commons.io.FilenameUtils;
import org.sonar.api.CoreProperties;
import org.sonar.api.batch.fs.InputFile;
import org.sonar.api.batch.fs.InputFile.Type;
@@ -32,7 +36,11 @@ import org.sonar.api.batch.fs.internal.DefaultInputFile;
import org.sonar.api.batch.fs.internal.DefaultInputModule;
import org.sonar.api.batch.fs.internal.DefaultInputProject;
import org.sonar.api.batch.fs.internal.SensorStrategy;
+import org.sonar.api.batch.scm.IgnoreCommand;
+import org.sonar.api.notifications.AnalysisWarnings;
import org.sonar.api.utils.MessageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.sonar.scanner.issue.ignore.scanner.IssueExclusionsLoader;
import org.sonar.scanner.repository.language.Language;
import org.sonar.scanner.scan.ScanProperties;
@@ -48,7 +56,10 @@ public class FileIndexer {
private static final Logger LOG = LoggerFactory.getLogger(FileIndexer.class);
+ private final AnalysisWarnings analysisWarnings;
private final ScanProperties properties;
+ private final InputFileFilter[] filters;
+ private final ProjectExclusionFilters projectExclusionFilters;
private final ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions;
private final IssueExclusionsLoader issueExclusionsLoader;
private final MetadataGenerator metadataGenerator;
@@ -60,13 +71,15 @@ public class FileIndexer {
private final StatusDetection statusDetection;
private final ScmChangedFiles scmChangedFiles;
- private final ModuleRelativePathWarner moduleRelativePathWarner;
- private final InputFileFilterRepository inputFileFilterRepository;
+ private boolean warnInclusionsAlreadyLogged;
+ private boolean warnExclusionsAlreadyLogged;
+ private boolean warnCoverageExclusionsAlreadyLogged;
+ private boolean warnDuplicationExclusionsAlreadyLogged;
public FileIndexer(DefaultInputProject project, ScannerComponentIdGenerator scannerComponentIdGenerator, InputComponentStore componentStore,
- ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, IssueExclusionsLoader issueExclusionsLoader,
- MetadataGenerator metadataGenerator, SensorStrategy sensorStrategy, LanguageDetection languageDetection, ScanProperties properties,
- ScmChangedFiles scmChangedFiles, StatusDetection statusDetection, ModuleRelativePathWarner moduleRelativePathWarner, InputFileFilterRepository inputFileFilterRepository) {
+ ProjectExclusionFilters projectExclusionFilters, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, IssueExclusionsLoader issueExclusionsLoader,
+ MetadataGenerator metadataGenerator, SensorStrategy sensorStrategy, LanguageDetection languageDetection, AnalysisWarnings analysisWarnings, ScanProperties properties,
+ InputFileFilter[] filters, ScmChangedFiles scmChangedFiles, StatusDetection statusDetection) {
this.project = project;
this.scannerComponentIdGenerator = scannerComponentIdGenerator;
this.componentStore = componentStore;
@@ -75,23 +88,55 @@ public class FileIndexer {
this.metadataGenerator = metadataGenerator;
this.sensorStrategy = sensorStrategy;
this.langDetection = languageDetection;
+ this.analysisWarnings = analysisWarnings;
this.properties = properties;
+ this.filters = filters;
+ this.projectExclusionFilters = projectExclusionFilters;
this.scmChangedFiles = scmChangedFiles;
this.statusDetection = statusDetection;
- this.moduleRelativePathWarner = moduleRelativePathWarner;
- this.inputFileFilterRepository = inputFileFilterRepository;
}
- void indexFile(DefaultInputModule module, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path sourceFile,
- Type type, ProgressReport progressReport) {
- Path projectRelativePath = project.getBaseDir().relativize(sourceFile);
- Path moduleRelativePath = module.getBaseDir().relativize(sourceFile);
+ void indexFile(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions,
+ Path sourceFile, Type type, ProgressReport progressReport, ProjectFileIndexer.ExclusionCounter exclusionCounter, @Nullable IgnoreCommand ignoreCommand)
+ throws IOException {
+ // get case of real file without resolving link
+ Path realAbsoluteFile = sourceFile.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize();
+ Path projectRelativePath = project.getBaseDir().relativize(realAbsoluteFile);
+ Path moduleRelativePath = module.getBaseDir().relativize(realAbsoluteFile);
+ boolean included = evaluateInclusionsFilters(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type);
+ if (!included) {
+ exclusionCounter.increaseByPatternsCount();
+ return;
+ }
+ boolean excluded = evaluateExclusionsFilters(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type);
+ if (excluded) {
+ exclusionCounter.increaseByPatternsCount();
+ return;
+ }
+ if (!realAbsoluteFile.startsWith(project.getBaseDir())) {
+ LOG.warn("File '{}' is ignored. It is not located in project basedir '{}'.", realAbsoluteFile.toAbsolutePath(), project.getBaseDir());
+ return;
+ }
+ if (!realAbsoluteFile.startsWith(module.getBaseDir())) {
+ LOG.warn("File '{}' is ignored. It is not located in module basedir '{}'.", realAbsoluteFile.toAbsolutePath(), module.getBaseDir());
+ return;
+ }
+
+ if (Files.exists(realAbsoluteFile) && isFileSizeBiggerThanLimit(realAbsoluteFile)) {
+ LOG.warn("File '{}' is bigger than {}MB and as consequence is removed from the analysis scope.", realAbsoluteFile.toAbsolutePath(), properties.fileSizeLimit());
+ return;
+ }
+
+ Language language = langDetection.language(realAbsoluteFile, projectRelativePath);
- // This should be fast; language should be cached from preprocessing step
- Language language = langDetection.language(sourceFile, projectRelativePath);
+ if (ignoreCommand != null && ignoreCommand.isIgnored(realAbsoluteFile)) {
+ LOG.debug("File '{}' is excluded by the scm ignore settings.", realAbsoluteFile);
+ exclusionCounter.increaseByScmCount();
+ return;
+ }
DefaultIndexedFile indexedFile = new DefaultIndexedFile(
- sourceFile,
+ realAbsoluteFile,
project.key(),
projectRelativePath.toString(),
moduleRelativePath.toString(),
@@ -99,7 +144,7 @@ public class FileIndexer {
language != null ? language.key() : null,
scannerComponentIdGenerator.getAsInt(),
sensorStrategy,
- scmChangedFiles.getOldRelativeFilePath(sourceFile)
+ scmChangedFiles.getOldRelativeFilePath(realAbsoluteFile)
);
DefaultInputFile inputFile = new DefaultInputFile(indexedFile, f -> metadataGenerator.setMetadata(module.key(), f, module.getEncoding()),
@@ -114,9 +159,7 @@ public class FileIndexer {
componentStore.put(module.key(), inputFile);
issueExclusionsLoader.addMulticriteriaPatterns(inputFile);
String langStr = inputFile.language() != null ? format("with language '%s'", inputFile.language()) : "with no language";
- if (LOG.isDebugEnabled()) {
- LOG.debug("'{}' indexed {}{}", projectRelativePath, type == Type.TEST ? "as test " : "", langStr);
- }
+ LOG.debug("'{}' indexed {}{}", projectRelativePath, type == Type.TEST ? "as test " : "", langStr);
evaluateCoverageExclusions(moduleCoverageAndDuplicationExclusions, inputFile);
evaluateDuplicationExclusions(moduleCoverageAndDuplicationExclusions, inputFile);
if (properties.preloadFileMetadata()) {
@@ -126,6 +169,42 @@ public class FileIndexer {
progressReport.message(count + " " + pluralizeFiles(count) + " indexed... (last one was " + inputFile.getProjectRelativePath() + ")");
}
+ private boolean evaluateInclusionsFilters(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectRelativePath, Path moduleRelativePath,
+ InputFile.Type type) {
+ if (!Arrays.equals(moduleExclusionFilters.getInclusionsConfig(type), projectExclusionFilters.getInclusionsConfig(type))) {
+ // Module specific configuration
+ return moduleExclusionFilters.isIncluded(realAbsoluteFile, moduleRelativePath, type);
+ }
+ boolean includedByProjectConfiguration = projectExclusionFilters.isIncluded(realAbsoluteFile, projectRelativePath, type);
+ if (includedByProjectConfiguration) {
+ return true;
+ } else if (moduleExclusionFilters.isIncluded(realAbsoluteFile, moduleRelativePath, type)) {
+ warnOnce(
+ type == Type.MAIN ? CoreProperties.PROJECT_INCLUSIONS_PROPERTY : CoreProperties.PROJECT_TEST_INCLUSIONS_PROPERTY,
+ FilenameUtils.normalize(projectRelativePath.toString(), true), () -> warnInclusionsAlreadyLogged, () -> warnInclusionsAlreadyLogged = true);
+ return true;
+ }
+ return false;
+ }
+
+ private boolean evaluateExclusionsFilters(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectRelativePath, Path moduleRelativePath,
+ InputFile.Type type) {
+ if (!Arrays.equals(moduleExclusionFilters.getExclusionsConfig(type), projectExclusionFilters.getExclusionsConfig(type))) {
+ // Module specific configuration
+ return moduleExclusionFilters.isExcluded(realAbsoluteFile, moduleRelativePath, type);
+ }
+ boolean includedByProjectConfiguration = projectExclusionFilters.isExcluded(realAbsoluteFile, projectRelativePath, type);
+ if (includedByProjectConfiguration) {
+ return true;
+ } else if (moduleExclusionFilters.isExcluded(realAbsoluteFile, moduleRelativePath, type)) {
+ warnOnce(
+ type == Type.MAIN ? CoreProperties.PROJECT_EXCLUSIONS_PROPERTY : CoreProperties.PROJECT_TEST_EXCLUSIONS_PROPERTY,
+ FilenameUtils.normalize(projectRelativePath.toString(), true), () -> warnExclusionsAlreadyLogged, () -> warnExclusionsAlreadyLogged = true);
+ return true;
+ }
+ return false;
+ }
+
private void checkIfAlreadyIndexed(DefaultInputFile inputFile) {
if (componentStore.inputFile(inputFile.getProjectRelativePath()) != null) {
throw MessageException.of("File " + inputFile + " can't be indexed twice. Please check that inclusion/exclusion patterns produce "
@@ -150,7 +229,8 @@ public class FileIndexer {
if (excludedByProjectConfiguration) {
return true;
} else if (moduleCoverageAndDuplicationExclusions.isExcludedForCoverage(inputFile)) {
- moduleRelativePathWarner.warnOnce(CoreProperties.PROJECT_COVERAGE_EXCLUSIONS_PROPERTY, inputFile.getProjectRelativePath());
+ warnOnce(CoreProperties.PROJECT_COVERAGE_EXCLUSIONS_PROPERTY, inputFile.getProjectRelativePath(), () -> warnCoverageExclusionsAlreadyLogged,
+ () -> warnCoverageExclusionsAlreadyLogged = true);
return true;
}
return false;
@@ -173,15 +253,26 @@ public class FileIndexer {
if (excludedByProjectConfiguration) {
return true;
} else if (moduleCoverageAndDuplicationExclusions.isExcludedForDuplication(inputFile)) {
- moduleRelativePathWarner.warnOnce(CoreProperties.CPD_EXCLUSIONS, inputFile.getProjectRelativePath());
+ warnOnce(CoreProperties.CPD_EXCLUSIONS, inputFile.getProjectRelativePath(), () -> warnDuplicationExclusionsAlreadyLogged,
+ () -> warnDuplicationExclusionsAlreadyLogged = true);
return true;
}
return false;
}
+ private void warnOnce(String propKey, String filePath, BooleanSupplier alreadyLoggedGetter, Runnable markAsLogged) {
+ if (!alreadyLoggedGetter.getAsBoolean()) {
+ String msg = "Specifying module-relative paths at project level in the property '" + propKey + "' is deprecated. " +
+ "To continue matching files like '" + filePath + "', update this property so that patterns refer to project-relative paths.";
+ LOG.warn(msg);
+ analysisWarnings.addUnique(msg);
+ markAsLogged.run();
+ }
+ }
+
private boolean accept(InputFile indexedFile) {
// InputFileFilter extensions. Might trigger generation of metadata
- for (InputFileFilter filter : inputFileFilterRepository.getInputFileFilters()) {
+ for (InputFileFilter filter : filters) {
if (!filter.accept(indexedFile)) {
LOG.debug("'{}' excluded by {}", indexedFile, filter.getClass().getName());
return false;
@@ -194,5 +285,7 @@ public class FileIndexer {
return count == 1 ? "file" : "files";
}
-
+ private boolean isFileSizeBiggerThanLimit(Path filePath) throws IOException {
+ return Files.size(filePath) > properties.fileSizeLimit() * 1024L * 1024L;
+ }
}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java
deleted file mode 100644
index 6faa87cc42a..00000000000
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2023 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.scanner.scan.filesystem;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.LinkOption;
-import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.Optional;
-import javax.annotation.CheckForNull;
-import org.apache.commons.io.FilenameUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.sonar.api.CoreProperties;
-import org.sonar.api.batch.fs.InputFile;
-import org.sonar.api.batch.fs.internal.DefaultInputModule;
-import org.sonar.api.batch.fs.internal.DefaultInputProject;
-import org.sonar.api.batch.scm.IgnoreCommand;
-import org.sonar.scanner.scan.ScanProperties;
-
-public class FilePreprocessor {
-
- private static final Logger LOG = LoggerFactory.getLogger(FilePreprocessor.class);
-
- private final ModuleRelativePathWarner moduleRelativePathWarner;
- private final DefaultInputProject project;
- private final LanguageDetection languageDetection;
- private final ProjectExclusionFilters projectExclusionFilters;
- private final ScanProperties properties;
-
- public FilePreprocessor(ModuleRelativePathWarner moduleRelativePathWarner, DefaultInputProject project,
- LanguageDetection languageDetection, ProjectExclusionFilters projectExclusionFilters, ScanProperties properties) {
- this.moduleRelativePathWarner = moduleRelativePathWarner;
- this.project = project;
- this.languageDetection = languageDetection;
- this.projectExclusionFilters = projectExclusionFilters;
- this.properties = properties;
- }
-
- public Optional<Path> processFile(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, Path sourceFile,
- InputFile.Type type, ProjectFilePreprocessor.ExclusionCounter exclusionCounter, @CheckForNull IgnoreCommand ignoreCommand) throws IOException {
- // get case of real file without resolving link
- Path realAbsoluteFile = sourceFile.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize();
- Path projectRelativePath = project.getBaseDir().relativize(realAbsoluteFile);
- Path moduleRelativePath = module.getBaseDir().relativize(realAbsoluteFile);
- boolean included = isFileIncluded(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type);
- if (!included) {
- exclusionCounter.increaseByPatternsCount();
- return Optional.empty();
- }
- boolean excluded = isFileExcluded(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type);
- if (excluded) {
- exclusionCounter.increaseByPatternsCount();
- return Optional.empty();
- }
-
- if (!realAbsoluteFile.startsWith(project.getBaseDir())) {
- LOG.warn("File '{}' is ignored. It is not located in project basedir '{}'.", realAbsoluteFile.toAbsolutePath(), project.getBaseDir());
- return Optional.empty();
- }
- if (!realAbsoluteFile.startsWith(module.getBaseDir())) {
- LOG.warn("File '{}' is ignored. It is not located in module basedir '{}'.", realAbsoluteFile.toAbsolutePath(), module.getBaseDir());
- return Optional.empty();
- }
-
- if (ignoreCommand != null && ignoreCommand.isIgnored(realAbsoluteFile)) {
- LOG.debug("File '{}' is excluded by the scm ignore settings.", realAbsoluteFile);
- exclusionCounter.increaseByScmCount();
- return Optional.empty();
- }
-
- if (Files.exists(realAbsoluteFile) && isFileSizeBiggerThanLimit(realAbsoluteFile)) {
- LOG.warn("File '{}' is bigger than {}MB and as consequence is removed from the analysis scope.", realAbsoluteFile.toAbsolutePath(), properties.fileSizeLimit());
- return Optional.empty();
- }
-
- languageDetection.language(realAbsoluteFile, projectRelativePath);
-
- return Optional.of(realAbsoluteFile);
- }
-
- private boolean isFileIncluded(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectRelativePath,
- Path moduleRelativePath, InputFile.Type type) {
- if (!Arrays.equals(moduleExclusionFilters.getInclusionsConfig(type), projectExclusionFilters.getInclusionsConfig(type))) {
- return moduleExclusionFilters.isIncluded(realAbsoluteFile, moduleRelativePath, type);
- }
- boolean includedByProjectConfiguration = projectExclusionFilters.isIncluded(realAbsoluteFile, projectRelativePath, type);
- if (includedByProjectConfiguration) {
- return true;
- }
- if (moduleExclusionFilters.isIncluded(realAbsoluteFile, moduleRelativePath, type)) {
- moduleRelativePathWarner.warnOnce(
- type == InputFile.Type.MAIN ? CoreProperties.PROJECT_INCLUSIONS_PROPERTY : CoreProperties.PROJECT_TEST_INCLUSIONS_PROPERTY,
- FilenameUtils.normalize(projectRelativePath.toString(), true));
- return true;
- }
- return false;
- }
-
- private boolean isFileExcluded(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectRelativePath,
- Path moduleRelativePath, InputFile.Type type) {
- if (!Arrays.equals(moduleExclusionFilters.getExclusionsConfig(type), projectExclusionFilters.getExclusionsConfig(type))) {
- return moduleExclusionFilters.isExcluded(realAbsoluteFile, moduleRelativePath, type);
- }
- boolean includedByProjectConfiguration = projectExclusionFilters.isExcluded(realAbsoluteFile, projectRelativePath, type);
- if (includedByProjectConfiguration) {
- return true;
- }
- if (moduleExclusionFilters.isExcluded(realAbsoluteFile, moduleRelativePath, type)) {
- moduleRelativePathWarner.warnOnce(
- type == InputFile.Type.MAIN ? CoreProperties.PROJECT_EXCLUSIONS_PROPERTY : CoreProperties.PROJECT_TEST_EXCLUSIONS_PROPERTY,
- FilenameUtils.normalize(projectRelativePath.toString(), true));
- return true;
- }
- return false;
- }
-
- private boolean isFileSizeBiggerThanLimit(Path filePath) throws IOException {
- return Files.size(filePath) > properties.fileSizeLimit() * 1024L * 1024L;
- }
-}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepository.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepository.java
deleted file mode 100644
index b6e14af85eb..00000000000
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepository.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2023 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.scanner.scan.filesystem;
-
-import org.sonar.api.batch.fs.InputFileFilter;
-
-public class InputFileFilterRepository {
- private final InputFileFilter[] inputFileFilters;
-
- public InputFileFilterRepository(InputFileFilter... inputFileFilters) {
- this.inputFileFilters = inputFileFilters;
- }
-
- public InputFileFilter[] getInputFileFilters() {
- return inputFileFilters;
- }
-}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java
index 24128133824..5ede0250c08 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java
@@ -25,7 +25,6 @@ import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.annotation.CheckForNull;
@@ -54,9 +53,8 @@ public class LanguageDetection {
*/
private final Map<Language, PathPattern[]> patternsByLanguage;
private final List<Language> languagesToConsider;
- private final Map<String, Language> languageCacheByPath;
- public LanguageDetection(Configuration settings, LanguagesRepository languages, Map<String, Language> languageCache) {
+ public LanguageDetection(Configuration settings, LanguagesRepository languages) {
Map<Language, PathPattern[]> patternsByLanguageBuilder = new LinkedHashMap<>();
for (Language language : languages.all()) {
String[] filePatterns = settings.getStringArray(getFileLangPatternPropKey(language.key()));
@@ -71,7 +69,6 @@ public class LanguageDetection {
languagesToConsider = List.copyOf(patternsByLanguageBuilder.keySet());
patternsByLanguage = unmodifiableMap(patternsByLanguageBuilder);
- languageCacheByPath = languageCache;
}
private static PathPattern[] getLanguagePatterns(Language language) {
@@ -92,16 +89,11 @@ public class LanguageDetection {
@CheckForNull
Language language(Path absolutePath, Path relativePath) {
- Language detectedLanguage = languageCacheByPath.get(absolutePath.toString());
- if (detectedLanguage != null) {
- return detectedLanguage;
- }
-
+ Language detectedLanguage = null;
for (Language language : languagesToConsider) {
if (isCandidateForLanguage(absolutePath, relativePath, language)) {
if (detectedLanguage == null) {
detectedLanguage = language;
- languageCacheByPath.put(absolutePath.toString(), language);
} else {
// Language was already forced by another pattern
throw MessageException.of(MessageFormat.format("Language of file ''{0}'' can not be decided as the file matches patterns of both {1} and {2}",
@@ -113,10 +105,6 @@ public class LanguageDetection {
return detectedLanguage;
}
- public Set<String> getDetectedLanguages() {
- return languageCacheByPath.values().stream().map(Language::key).collect(Collectors.toSet());
- }
-
private boolean isCandidateForLanguage(Path absolutePath, Path relativePath, Language language) {
PathPattern[] patterns = patternsByLanguage.get(language);
return patterns != null && Arrays.stream(patterns).anyMatch(pattern -> pattern.match(absolutePath, relativePath, false));
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleRelativePathWarner.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleRelativePathWarner.java
deleted file mode 100644
index 766955e438c..00000000000
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleRelativePathWarner.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2023 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.scanner.scan.filesystem;
-
-import java.util.HashSet;
-import java.util.Set;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.sonar.api.notifications.AnalysisWarnings;
-
-public class ModuleRelativePathWarner {
-
- private static final Logger LOG = LoggerFactory.getLogger(ModuleRelativePathWarner.class);
- private final AnalysisWarnings analysisWarnings;
- private final Set<String> previouslyWarnedProps = new HashSet<>();
-
- public ModuleRelativePathWarner(AnalysisWarnings analysisWarnings) {
- this.analysisWarnings = analysisWarnings;
- }
-
- public void warnOnce(String propKey, String filePath) {
- if (!previouslyWarnedProps.contains(propKey)) {
- previouslyWarnedProps.add(propKey);
- String msg = "Specifying module-relative paths at project level in the property '" + propKey + "' is deprecated. " +
- "To continue matching files like '" + filePath + "', update this property so that patterns refer to project-relative paths.";
- LOG.warn(msg);
- analysisWarnings.addUnique(msg);
- }
- }
-}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java
index 5ff7ca1ef17..35d5bf3fdb4 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java
@@ -20,22 +20,33 @@
package org.sonar.scanner.scan.filesystem;
import java.io.IOException;
+import java.nio.file.AccessDeniedException;
+import java.nio.file.FileSystemLoopException;
import java.nio.file.FileVisitOption;
+import java.nio.file.FileVisitResult;
+import java.nio.file.FileVisitor;
import java.nio.file.Files;
+import java.nio.file.LinkOption;
import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.DosFileAttributes;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.commons.lang.SystemUtils;
+import org.sonar.api.batch.fs.InputFile;
import org.sonar.api.batch.fs.InputFile.Type;
import org.sonar.api.batch.fs.internal.DefaultInputModule;
+import org.sonar.api.batch.scm.IgnoreCommand;
import org.sonar.api.notifications.AnalysisWarnings;
import org.sonar.api.scan.filesystem.PathResolver;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.sonar.scanner.bootstrap.GlobalConfiguration;
import org.sonar.scanner.bootstrap.GlobalServerSettings;
import org.sonar.scanner.fs.InputModuleHierarchy;
@@ -43,8 +54,12 @@ import org.sonar.scanner.scan.ModuleConfiguration;
import org.sonar.scanner.scan.ModuleConfigurationProvider;
import org.sonar.scanner.scan.ProjectServerSettings;
import org.sonar.scanner.scan.SonarGlobalPropertiesFilter;
+import org.sonar.scanner.scm.ScmConfiguration;
import org.sonar.scanner.util.ProgressReport;
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+
/**
* Index project input files into {@link InputComponentStore}.
*/
@@ -54,13 +69,15 @@ public class ProjectFileIndexer {
private final ProjectExclusionFilters projectExclusionFilters;
private final SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter;
private final ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions;
+ private final ScmConfiguration scmConfiguration;
private final InputComponentStore componentStore;
private final InputModuleHierarchy inputModuleHierarchy;
private final GlobalConfiguration globalConfig;
private final GlobalServerSettings globalServerSettings;
private final ProjectServerSettings projectServerSettings;
private final FileIndexer fileIndexer;
- private final ProjectFilePreprocessor projectFilePreprocessor;
+ private final IgnoreCommand ignoreCommand;
+ private final boolean useScmExclusion;
private final AnalysisWarnings analysisWarnings;
private ProgressReport progressReport;
@@ -68,8 +85,8 @@ public class ProjectFileIndexer {
public ProjectFileIndexer(InputComponentStore componentStore, ProjectExclusionFilters exclusionFilters,
SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter, InputModuleHierarchy inputModuleHierarchy,
GlobalConfiguration globalConfig, GlobalServerSettings globalServerSettings, ProjectServerSettings projectServerSettings,
- FileIndexer fileIndexer, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions,
- ProjectFilePreprocessor projectFilePreprocessor, AnalysisWarnings analysisWarnings) {
+ FileIndexer fileIndexer, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, ScmConfiguration scmConfiguration,
+ AnalysisWarnings analysisWarnings) {
this.componentStore = componentStore;
this.sonarGlobalPropertiesFilter = sonarGlobalPropertiesFilter;
this.inputModuleHierarchy = inputModuleHierarchy;
@@ -79,8 +96,10 @@ public class ProjectFileIndexer {
this.fileIndexer = fileIndexer;
this.projectExclusionFilters = exclusionFilters;
this.projectCoverageAndDuplicationExclusions = projectCoverageAndDuplicationExclusions;
- this.projectFilePreprocessor = projectFilePreprocessor;
+ this.scmConfiguration = scmConfiguration;
this.analysisWarnings = analysisWarnings;
+ this.ignoreCommand = loadIgnoreCommand();
+ this.useScmExclusion = ignoreCommand != null;
}
public void index() {
@@ -89,22 +108,47 @@ public class ProjectFileIndexer {
LOG.info("Project configuration:");
projectExclusionFilters.log(" ");
projectCoverageAndDuplicationExclusions.log(" ");
+ ExclusionCounter exclusionCounter = new ExclusionCounter();
- indexModulesRecursively(inputModuleHierarchy.root());
+ if (useScmExclusion) {
+ ignoreCommand.init(inputModuleHierarchy.root().getBaseDir().toAbsolutePath());
+ indexModulesRecursively(inputModuleHierarchy.root(), exclusionCounter);
+ ignoreCommand.clean();
+ } else {
+ indexModulesRecursively(inputModuleHierarchy.root(), exclusionCounter);
+ }
int totalIndexed = componentStore.inputFiles().size();
progressReport.stop(totalIndexed + " " + pluralizeFiles(totalIndexed) + " indexed");
+ int excludedFileByPatternCount = exclusionCounter.getByPatternsCount();
+ if (projectExclusionFilters.hasPattern() || excludedFileByPatternCount > 0) {
+ LOG.info("{} {} ignored because of inclusion/exclusion patterns", excludedFileByPatternCount, pluralizeFiles(excludedFileByPatternCount));
+ }
+ int excludedFileByScmCount = exclusionCounter.getByScmCount();
+ if (useScmExclusion) {
+ LOG.info("{} {} ignored because of scm ignore settings", excludedFileByScmCount, pluralizeFiles(excludedFileByScmCount));
+ }
}
- private void indexModulesRecursively(DefaultInputModule module) {
- inputModuleHierarchy.children(module).stream()
- .sorted(Comparator.comparing(DefaultInputModule::key))
- .forEach(this::indexModulesRecursively);
- index(module);
+ private IgnoreCommand loadIgnoreCommand() {
+ try {
+ if (!scmConfiguration.isExclusionDisabled() && scmConfiguration.provider() != null) {
+ return scmConfiguration.provider().ignoreCommand();
+ }
+ } catch (UnsupportedOperationException e) {
+ LOG.debug("File exclusion based on SCM ignore information is not available with this plugin.");
+ }
+
+ return null;
}
- private void index(DefaultInputModule module) {
+ private void indexModulesRecursively(DefaultInputModule module, ExclusionCounter exclusionCounter) {
+ inputModuleHierarchy.children(module).stream().sorted(Comparator.comparing(DefaultInputModule::key)).forEach(m -> indexModulesRecursively(m, exclusionCounter));
+ index(module, exclusionCounter);
+ }
+
+ private void index(DefaultInputModule module, ExclusionCounter exclusionCounter) {
// Emulate creation of module level settings
ModuleConfiguration moduleConfig = new ModuleConfigurationProvider(sonarGlobalPropertiesFilter).provide(globalConfig, module, globalServerSettings, projectServerSettings);
ModuleExclusionFilters moduleExclusionFilters = new ModuleExclusionFilters(moduleConfig, analysisWarnings);
@@ -117,10 +161,13 @@ public class ProjectFileIndexer {
moduleExclusionFilters.log(" ");
moduleCoverageAndDuplicationExclusions.log(" ");
}
- List<Path> mainSourceDirsOrFiles = projectFilePreprocessor.getMainSourcesByModule(module);
- indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, mainSourceDirsOrFiles, Type.MAIN);
- projectFilePreprocessor.getTestSourcesByModule(module)
- .ifPresent(tests -> indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, tests, Type.TEST));
+ boolean hasChildModules = !module.definition().getSubProjects().isEmpty();
+ boolean hasTests = module.getTestDirsOrFiles().isPresent();
+ // Default to index basedir when no sources provided
+ List<Path> mainSourceDirsOrFiles = module.getSourceDirsOrFiles()
+ .orElseGet(() -> hasChildModules || hasTests ? emptyList() : singletonList(module.getBaseDir().toAbsolutePath()));
+ indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, mainSourceDirsOrFiles, Type.MAIN, exclusionCounter);
+ module.getTestDirsOrFiles().ifPresent(tests -> indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, tests, Type.TEST, exclusionCounter));
}
private static void logPaths(String label, Path baseDir, List<Path> paths) {
@@ -129,7 +176,7 @@ public class ProjectFileIndexer {
for (Iterator<Path> it = paths.iterator(); it.hasNext(); ) {
Path file = it.next();
Optional<String> relativePathToBaseDir = PathResolver.relativize(baseDir, file);
- if (relativePathToBaseDir.isEmpty()) {
+ if (!relativePathToBaseDir.isPresent()) {
sb.append(file);
} else if (StringUtils.isBlank(relativePathToBaseDir.get())) {
sb.append(".");
@@ -148,14 +195,19 @@ public class ProjectFileIndexer {
}
}
- private void indexFiles(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions,
- List<Path> sources, Type type) {
+ private static String pluralizeFiles(int count) {
+ return count == 1 ? "file" : "files";
+ }
+
+ private void indexFiles(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters,
+ ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, List<Path> sources, Type type, ExclusionCounter exclusionCounter) {
try {
for (Path dirOrFile : sources) {
if (dirOrFile.toFile().isDirectory()) {
- indexDirectory(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type);
+ indexDirectory(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type, exclusionCounter);
} else {
- fileIndexer.indexFile(module, moduleCoverageAndDuplicationExclusions, dirOrFile, type, progressReport);
+ fileIndexer.indexFile(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type, progressReport, exclusionCounter,
+ ignoreCommand);
}
}
} catch (IOException e) {
@@ -164,17 +216,141 @@ public class ProjectFileIndexer {
}
private void indexDirectory(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters,
- ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions,
- Path dirToIndex, Type type) throws IOException {
+ ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path dirToIndex, Type type, ExclusionCounter exclusionCounter)
+ throws IOException {
Files.walkFileTree(dirToIndex.normalize(), Collections.singleton(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE,
- new DirectoryFileVisitor(file -> fileIndexer.indexFile(module, moduleCoverageAndDuplicationExclusions, file, type, progressReport),
- module, moduleExclusionFilters, inputModuleHierarchy, type));
+ new IndexFileVisitor(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, type, exclusionCounter));
}
- private static String pluralizeFiles(int count) {
- return count == 1 ? "file" : "files";
+
+ /**
+ * <p>Checks if the path is a directory that is excluded.</p>
+ *
+ * <p>Exclusions patterns are checked both at project and module level.</p>
+ *
+ * @param moduleExclusionFilters The exclusion filters.
+ * @param realAbsoluteFile The path to be checked.
+ * @param projectBaseDir The project base directory.
+ * @param moduleBaseDir The module base directory.
+ * @param type The input file type.
+ * @return True if path is an excluded directory, false otherwise.
+ */
+ private static boolean isExcludedDirectory(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectBaseDir, Path moduleBaseDir,
+ InputFile.Type type) {
+ Path projectRelativePath = projectBaseDir.relativize(realAbsoluteFile);
+ Path moduleRelativePath = moduleBaseDir.relativize(realAbsoluteFile);
+ return moduleExclusionFilters.isExcludedAsParentDirectoryOfExcludedChildren(realAbsoluteFile, projectRelativePath, projectBaseDir, type)
+ || moduleExclusionFilters.isExcludedAsParentDirectoryOfExcludedChildren(realAbsoluteFile, moduleRelativePath, moduleBaseDir, type);
+ }
+
+ private class IndexFileVisitor implements FileVisitor<Path> {
+ private final DefaultInputModule module;
+ private final ModuleExclusionFilters moduleExclusionFilters;
+ private final ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions;
+ private final Type type;
+ private final ExclusionCounter exclusionCounter;
+
+ IndexFileVisitor(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions,
+ Type type,
+ ExclusionCounter exclusionCounter) {
+ this.module = module;
+ this.moduleExclusionFilters = moduleExclusionFilters;
+ this.moduleCoverageAndDuplicationExclusions = moduleCoverageAndDuplicationExclusions;
+ this.type = type;
+ this.exclusionCounter = exclusionCounter;
+ }
+
+ @Override
+ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+ if (isHidden(dir)) {
+ return FileVisitResult.SKIP_SUBTREE;
+ }
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+ if (!Files.isHidden(file)) {
+ fileIndexer.indexFile(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, file, type, progressReport, exclusionCounter, ignoreCommand);
+ }
+ return FileVisitResult.CONTINUE;
+ }
+
+ /**
+ * <p>Overridden method to handle exceptions while visiting files in the analysis.</p>
+ *
+ * <p>
+ * <ul>
+ * <li>FileSystemLoopException - We show a warning that a symlink loop exists and we skip the file.</li>
+ * <li>AccessDeniedException for excluded files/directories - We skip the file, as files excluded from the analysis, shouldn't throw access exceptions.</li>
+ * </ul>
+ * </p>
+ *
+ * @param file a reference to the file
+ * @param exc the I/O exception that prevented the file from being visited
+ * @throws IOException
+ */
+ @Override
+ public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
+ if (exc instanceof FileSystemLoopException) {
+ LOG.warn("Not indexing due to symlink loop: {}", file.toFile());
+ return FileVisitResult.CONTINUE;
+ } else if (exc instanceof AccessDeniedException && isExcluded(file)) {
+ return FileVisitResult.CONTINUE;
+ }
+ throw exc;
+ }
+
+ /**
+ * <p>Checks if the directory is excluded in the analysis or not. Only the exclusions are checked.</p>
+ *
+ * <p>The inclusions cannot be checked for directories, since the current implementation of pattern matching is intended only for files.</p>
+ *
+ * @param path The file or directory.
+ * @return True if file/directory is excluded from the analysis, false otherwise.
+ */
+ private boolean isExcluded(Path path) throws IOException {
+ Path realAbsoluteFile = path.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize();
+ return isExcludedDirectory(moduleExclusionFilters, realAbsoluteFile, inputModuleHierarchy.root().getBaseDir(), module.getBaseDir(), type);
+ }
+
+ @Override
+ public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
+ return FileVisitResult.CONTINUE;
+ }
+
+ private boolean isHidden(Path path) throws IOException {
+ if (SystemUtils.IS_OS_WINDOWS) {
+ try {
+ DosFileAttributes dosFileAttributes = Files.readAttributes(path, DosFileAttributes.class, LinkOption.NOFOLLOW_LINKS);
+ return dosFileAttributes.isHidden();
+ } catch (UnsupportedOperationException e) {
+ return path.toFile().isHidden();
+ }
+ } else {
+ return Files.isHidden(path);
+ }
+ }
}
+ static class ExclusionCounter {
+ private final AtomicInteger excludedByPatternsCount = new AtomicInteger(0);
+ private final AtomicInteger excludedByScmCount = new AtomicInteger(0);
+
+ public void increaseByPatternsCount() {
+ excludedByPatternsCount.incrementAndGet();
+ }
+
+ public int getByPatternsCount() {
+ return excludedByPatternsCount.get();
+ }
+ public void increaseByScmCount() {
+ excludedByScmCount.incrementAndGet();
+ }
+ public int getByScmCount() {
+ return excludedByScmCount.get();
+ }
+ }
}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java
deleted file mode 100644
index 54d4f2b8d6c..00000000000
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java
+++ /dev/null
@@ -1,230 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2023 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.scanner.scan.filesystem;
-
-import java.io.IOException;
-import java.nio.file.FileVisitOption;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.concurrent.TimeUnit;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.sonar.api.batch.fs.InputFile;
-import org.sonar.api.batch.fs.internal.DefaultInputModule;
-import org.sonar.api.batch.scm.IgnoreCommand;
-import org.sonar.api.batch.scm.ScmProvider;
-import org.sonar.api.notifications.AnalysisWarnings;
-import org.sonar.scanner.bootstrap.GlobalConfiguration;
-import org.sonar.scanner.bootstrap.GlobalServerSettings;
-import org.sonar.scanner.fs.InputModuleHierarchy;
-import org.sonar.scanner.scan.ModuleConfiguration;
-import org.sonar.scanner.scan.ModuleConfigurationProvider;
-import org.sonar.scanner.scan.ProjectServerSettings;
-import org.sonar.scanner.scan.SonarGlobalPropertiesFilter;
-import org.sonar.scanner.scm.ScmConfiguration;
-import org.sonar.scanner.util.ProgressReport;
-
-import static java.util.Collections.emptyList;
-import static java.util.Collections.singletonList;
-
-public class ProjectFilePreprocessor {
-
- private static final Logger LOG = LoggerFactory.getLogger(ProjectFilePreprocessor.class);
- private static final String TELEMETRY_STEP_NAME = "file.preprocessing";
-
- private final AnalysisWarnings analysisWarnings;
- private final IgnoreCommand ignoreCommand;
- private final boolean useScmExclusion;
- private final ScmConfiguration scmConfiguration;
- private final InputModuleHierarchy inputModuleHierarchy;
- private final GlobalConfiguration globalConfig;
- private final GlobalServerSettings globalServerSettings;
- private final ProjectServerSettings projectServerSettings;
- private final LanguageDetection languageDetection;
- private final FilePreprocessor filePreprocessor;
- private final ProjectExclusionFilters projectExclusionFilters;
-
- private final SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter;
-
- private final Map<DefaultInputModule, List<Path>> mainSourcesByModule = new HashMap<>();
- private final Map<DefaultInputModule, List<Path>> testSourcesByModule = new HashMap<>();
-
- private int totalFilesPreprocessed = 0;
-
- public ProjectFilePreprocessor(AnalysisWarnings analysisWarnings, ScmConfiguration scmConfiguration, InputModuleHierarchy inputModuleHierarchy,
- GlobalConfiguration globalConfig, GlobalServerSettings globalServerSettings, ProjectServerSettings projectServerSettings,
- LanguageDetection languageDetection, FilePreprocessor filePreprocessor,
- ProjectExclusionFilters projectExclusionFilters, SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter) {
- this.analysisWarnings = analysisWarnings;
- this.scmConfiguration = scmConfiguration;
- this.inputModuleHierarchy = inputModuleHierarchy;
- this.globalConfig = globalConfig;
- this.globalServerSettings = globalServerSettings;
- this.projectServerSettings = projectServerSettings;
- this.languageDetection = languageDetection;
- this.filePreprocessor = filePreprocessor;
- this.projectExclusionFilters = projectExclusionFilters;
- this.sonarGlobalPropertiesFilter = sonarGlobalPropertiesFilter;
- this.ignoreCommand = loadIgnoreCommand();
- this.useScmExclusion = ignoreCommand != null;
- }
-
- public void execute() {
- ProgressReport progressReport = new ProgressReport("Report about progress of file preprocessing",
- TimeUnit.SECONDS.toMillis(10));
- progressReport.start("Preprocessing files...");
- ExclusionCounter exclusionCounter = new ExclusionCounter();
-
- if (useScmExclusion) {
- ignoreCommand.init(inputModuleHierarchy.root().getBaseDir().toAbsolutePath());
- processModulesRecursively(inputModuleHierarchy.root(), exclusionCounter);
- ignoreCommand.clean();
- } else {
- processModulesRecursively(inputModuleHierarchy.root(), exclusionCounter);
- }
-
- int totalLanguagesDetected = languageDetection.getDetectedLanguages().size();
-
- progressReport.stop(String.format("%s detected in %s", pluralizeWithCount("language", totalLanguagesDetected),
- pluralizeWithCount("preprocessed file", totalFilesPreprocessed)));
-
- int excludedFileByPatternCount = exclusionCounter.getByPatternsCount();
- if (projectExclusionFilters.hasPattern() || excludedFileByPatternCount > 0) {
- if (LOG.isInfoEnabled()) {
- LOG.info("{} ignored because of inclusion/exclusion patterns", pluralizeWithCount("file", excludedFileByPatternCount));
- }
- }
-
- int excludedFileByScmCount = exclusionCounter.getByScmCount();
- if (useScmExclusion) {
- if (LOG.isInfoEnabled()) {
- LOG.info("{} ignored because of scm ignore settings", pluralizeWithCount("file", excludedFileByScmCount));
- }
- }
- }
-
- private void processModulesRecursively(DefaultInputModule module, ExclusionCounter exclusionCounter) {
- inputModuleHierarchy.children(module).stream().sorted(Comparator.comparing(DefaultInputModule::key)).forEach(
- m -> processModulesRecursively(m, exclusionCounter));
- processModule(module, exclusionCounter);
- }
-
- private void processModule(DefaultInputModule module, ExclusionCounter exclusionCounter) {
- // Emulate creation of module level settings
- ModuleConfiguration moduleConfig = new ModuleConfigurationProvider(sonarGlobalPropertiesFilter).provide(globalConfig, module, globalServerSettings, projectServerSettings);
- ModuleExclusionFilters moduleExclusionFilters = new ModuleExclusionFilters(moduleConfig, analysisWarnings);
- boolean hasChildModules = !module.definition().getSubProjects().isEmpty();
- boolean hasTests = module.getTestDirsOrFiles().isPresent();
- // Default to index basedir when no sources provided
- List<Path> mainSourceDirsOrFiles = module.getSourceDirsOrFiles()
- .orElseGet(() -> hasChildModules || hasTests ? emptyList() : singletonList(module.getBaseDir().toAbsolutePath()));
- List<Path> processedSources = processModuleSources(module, moduleExclusionFilters, mainSourceDirsOrFiles, InputFile.Type.MAIN,
- exclusionCounter);
- mainSourcesByModule.put(module, processedSources);
- totalFilesPreprocessed += processedSources.size();
- module.getTestDirsOrFiles().ifPresent(tests -> {
- List<Path> processedTestSources = processModuleSources(module, moduleExclusionFilters, tests, InputFile.Type.TEST, exclusionCounter);
- testSourcesByModule.put(module, processedTestSources);
- totalFilesPreprocessed += processedTestSources.size();
- });
- }
-
- private List<Path> processModuleSources(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, List<Path> sources,
- InputFile.Type type, ExclusionCounter exclusionCounter) {
- List<Path> processedFiles = new ArrayList<>();
- try {
- for (Path dirOrFile : sources) {
- if (dirOrFile.toFile().isDirectory()) {
- processedFiles.addAll(processDirectory(module, moduleExclusionFilters, dirOrFile, type, exclusionCounter));
- } else {
- filePreprocessor.processFile(module, moduleExclusionFilters, dirOrFile, type, exclusionCounter, ignoreCommand)
- .ifPresent(processedFiles::add);
- }
- }
- } catch (IOException e) {
- throw new IllegalStateException("Failed to preprocess files", e);
- }
- return processedFiles;
- }
-
- private List<Path> processDirectory(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, Path path,
- InputFile.Type type, ExclusionCounter exclusionCounter) throws IOException {
- List<Path> processedFiles = new ArrayList<>();
- Files.walkFileTree(path.normalize(), Collections.singleton(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE,
- new DirectoryFileVisitor(file -> filePreprocessor.processFile(module, moduleExclusionFilters, file, type, exclusionCounter,
- ignoreCommand).ifPresent(processedFiles::add), module, moduleExclusionFilters, inputModuleHierarchy, type)
- );
- return processedFiles;
- }
-
- public List<Path> getMainSourcesByModule(DefaultInputModule module) {
- return Collections.unmodifiableList(mainSourcesByModule.get(module));
- }
-
- public Optional<List<Path>> getTestSourcesByModule(DefaultInputModule module) {
- return Optional.ofNullable(testSourcesByModule.get(module)).map(Collections::unmodifiableList);
- }
-
- private IgnoreCommand loadIgnoreCommand() {
- try {
- ScmProvider provider = scmConfiguration.provider();
- if (!scmConfiguration.isExclusionDisabled() && provider != null) {
- return provider.ignoreCommand();
- }
- } catch (UnsupportedOperationException e) {
- LOG.debug("File exclusion based on SCM ignore information is not available with this plugin.");
- }
-
- return null;
- }
-
- private static String pluralizeWithCount(String str, int count) {
- String pluralized = count == 1 ? str : (str + "s");
- return count + " " + pluralized;
- }
-
- public static class ExclusionCounter {
- private int excludedByPatternsCount = 0;
- private int excludedByScmCount = 0;
-
- public void increaseByPatternsCount() {
- excludedByPatternsCount++;
- }
-
- public int getByPatternsCount() {
- return excludedByPatternsCount;
- }
-
- public void increaseByScmCount() {
- excludedByScmCount++;
- }
-
- public int getByScmCount() {
- return excludedByScmCount;
- }
- }
-}
diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitorTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitorTest.java
deleted file mode 100644
index fd79238f65d..00000000000
--- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitorTest.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2023 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.scanner.scan.filesystem;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.FileSystemLoopException;
-import java.nio.file.FileVisitResult;
-import java.nio.file.Files;
-import java.nio.file.LinkOption;
-import java.nio.file.Path;
-import java.nio.file.attribute.BasicFileAttributes;
-import org.apache.commons.lang.SystemUtils;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.sonar.api.batch.fs.InputFile;
-import org.sonar.api.batch.fs.internal.DefaultInputModule;
-import org.sonar.scanner.fs.InputModuleHierarchy;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.assertThrows;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.verify;
-
-public class DirectoryFileVisitorTest {
-
- @ClassRule
- public static TemporaryFolder temp = new TemporaryFolder();
-
- private final DefaultInputModule module = mock();
- private final ModuleExclusionFilters moduleExclusionFilters = mock();
- private final InputModuleHierarchy inputModuleHierarchy = mock();
- private final InputFile.Type type = mock();
-
- @Test
- public void visit_hidden_file() throws IOException {
- DirectoryFileVisitor.FileVisitAction action = mock(DirectoryFileVisitor.FileVisitAction.class);
-
- File hidden = temp.newFile(".hidden");
- if (SystemUtils.IS_OS_WINDOWS) {
- Files.setAttribute(hidden.toPath(), "dos:hidden", true, LinkOption.NOFOLLOW_LINKS);
- }
-
-
- DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleExclusionFilters, inputModuleHierarchy, type);
- underTest.visitFile(hidden.toPath(), Files.readAttributes(hidden.toPath(), BasicFileAttributes.class));
-
- verify(action, never()).execute(any(Path.class));
- }
-
- @Test
- public void test_visit_file_failed_generic_io_exception() throws IOException {
- DirectoryFileVisitor.FileVisitAction action = mock(DirectoryFileVisitor.FileVisitAction.class);
-
- File file = temp.newFile("failed");
-
- DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleExclusionFilters, inputModuleHierarchy, type);
- assertThrows(IOException.class, () -> underTest.visitFileFailed(file.toPath(), new IOException()));
- }
-
- @Test
- public void test_visit_file_failed_file_system_loop_exception() throws IOException {
- DirectoryFileVisitor.FileVisitAction action = mock(DirectoryFileVisitor.FileVisitAction.class);
-
- File file = temp.newFile("symlink");
-
- DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleExclusionFilters, inputModuleHierarchy, type);
- FileVisitResult result = underTest.visitFileFailed(file.toPath(), new FileSystemLoopException(file.getPath()));
-
- assertThat(result).isEqualTo(FileVisitResult.CONTINUE);
- }
-
-}
diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepositoryTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepositoryTest.java
deleted file mode 100644
index 12f1cb0c63a..00000000000
--- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepositoryTest.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2023 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.scanner.scan.filesystem;
-
-import org.junit.Test;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class InputFileFilterRepositoryTest {
-
- @Test
- public void should_not_return_null_if_initialized_with_no_filters() {
- InputFileFilterRepository underTest = new InputFileFilterRepository();
- assertThat(underTest.getInputFileFilters()).isNotNull();
- }
-
- @Test
- public void should_return_filters_from_initialization() {
- InputFileFilterRepository underTest = new InputFileFilterRepository(f -> true);
- assertThat(underTest.getInputFileFilters()).isNotNull();
- assertThat(underTest.getInputFileFilters()).hasSize(1);
- }
-
-}
diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java
index 18408451557..e5810baba5a 100644
--- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java
+++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java
@@ -24,8 +24,6 @@ import com.tngtech.java.junit.dataprovider.DataProviderRunner;
import com.tngtech.java.junit.dataprovider.UseDataProvider;
import java.io.File;
import java.nio.file.Paths;
-import java.util.HashMap;
-import java.util.Map;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -40,11 +38,7 @@ import org.sonar.scanner.repository.language.LanguagesRepository;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.endsWith;
import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
@RunWith(DataProviderRunner.class)
public class LanguageDetectionTest {
@@ -78,7 +72,7 @@ public class LanguageDetectionTest {
@Test
public void detectLanguageKey_shouldDetectByFileExtension() {
LanguagesRepository languages = new FakeLanguagesRepository(new Languages(new MockLanguage("java", "java", "jav"), new MockLanguage("cobol", "cbl", "cob")));
- LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>());
+ LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
assertThat(detectLanguageKey(detection, "Foo.java")).isEqualTo("java");
assertThat(detectLanguageKey(detection, "src/Foo.java")).isEqualTo("java");
@@ -100,7 +94,7 @@ public class LanguageDetectionTest {
new MockLanguage("docker", new String[0], new String[] {"*.dockerfile", "*.Dockerfile", "Dockerfile", "Dockerfile.*"}),
new MockLanguage("terraform", new String[] {"tf"}, new String[] {".tf"}),
new MockLanguage("java", new String[0], new String[] {"**/*Test.java"})));
- LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>());
+ LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
assertThat(detectLanguageKey(detection, fileName)).isEqualTo(expectedLanguageKey);
}
@@ -123,7 +117,7 @@ public class LanguageDetectionTest {
@Test
public void detectLanguageKey_shouldNotFailIfNoLanguage() {
- LanguageDetection detection = spy(new LanguageDetection(settings.asConfig(), new FakeLanguagesRepository(new Languages()), new HashMap<>()));
+ LanguageDetection detection = spy(new LanguageDetection(settings.asConfig(), new FakeLanguagesRepository(new Languages())));
assertThat(detectLanguageKey(detection, "Foo.java")).isNull();
}
@@ -131,14 +125,14 @@ public class LanguageDetectionTest {
public void detectLanguageKey_shouldAllowPluginsToDeclareFileExtensionTwiceForCaseSensitivity() {
LanguagesRepository languages = new FakeLanguagesRepository(new Languages(new MockLanguage("abap", "abap", "ABAP")));
- LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>());
+ LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
assertThat(detectLanguageKey(detection, "abc.abap")).isEqualTo("abap");
}
@Test
public void detectLanguageKey_shouldFailIfConflictingLanguageSuffix() {
LanguagesRepository languages = new FakeLanguagesRepository(new Languages(new MockLanguage("xml", "xhtml"), new MockLanguage("web", "xhtml")));
- LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>());
+ LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
assertThatThrownBy(() -> detectLanguageKey(detection, "abc.xhtml"))
.isInstanceOf(MessageException.class)
.hasMessageContaining("Language of file 'abc.xhtml' can not be decided as the file matches patterns of both ")
@@ -152,7 +146,7 @@ public class LanguageDetectionTest {
settings.setProperty("sonar.lang.patterns.xml", "xml/**");
settings.setProperty("sonar.lang.patterns.web", "web/**");
- LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>());
+ LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
assertThat(detectLanguageKey(detection, "xml/abc.xhtml")).isEqualTo("xml");
assertThat(detectLanguageKey(detection, "web/abc.xhtml")).isEqualTo("web");
}
@@ -163,7 +157,7 @@ public class LanguageDetectionTest {
settings.setProperty("sonar.lang.patterns.abap", "*.abap,*.txt");
settings.setProperty("sonar.lang.patterns.cobol", "*.cobol,*.txt");
- LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>());
+ LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
assertThat(detectLanguageKey(detection, "abc.abap")).isEqualTo("abap");
assertThat(detectLanguageKey(detection, "abc.cobol")).isEqualTo("cobol");
@@ -174,19 +168,6 @@ public class LanguageDetectionTest {
.hasMessageContaining("sonar.lang.patterns.cobol : *.cobol,*.txt");
}
- @Test
- public void should_cache_detected_language_by_file_path() {
- Map<String, org.sonar.scanner.repository.language.Language> languageCacheSpy = spy(new HashMap<>());
- LanguagesRepository languages = new FakeLanguagesRepository(new Languages(
- new MockLanguage("java", "java", "jav"), new MockLanguage("cobol", "cbl", "cob")));
- LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, languageCacheSpy);
-
- assertThat(detectLanguageKey(detection, "Foo.java")).isEqualTo("java");
- assertThat(detectLanguageKey(detection, "Foo.java")).isEqualTo("java");
- verify(languageCacheSpy, times(1)).put(endsWith("/Foo.java"), any(org.sonar.scanner.repository.language.Language.class));
- verify(languageCacheSpy, times(2)).get(endsWith("/Foo.java"));
- }
-
private String detectLanguageKey(LanguageDetection detection, String path) {
org.sonar.scanner.repository.language.Language language = detection.language(new File(temp.getRoot(), path).toPath(), Paths.get(path));
return language != null ? language.key() : null;
diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/sonar-project.properties b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/sonar-project.properties
deleted file mode 100644
index 57c2c062b0e..00000000000
--- a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/sonar-project.properties
+++ /dev/null
@@ -1,5 +0,0 @@
-sonar.organization=org1
-sonar.projectKey=sample-with-empty-file
-sonar.projectName=Sample With Empty
-sonar.projectVersion=0.1-SNAPSHOT
-sonar.sources=xources
diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/HelloJava.xoo b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/HelloJava.xoo
deleted file mode 100644
index ee9bf789a53..00000000000
--- a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/HelloJava.xoo
+++ /dev/null
@@ -1,8 +0,0 @@
-package hello;
-
-public class HelloJava {
-
- public static void main(String[] args) {
- System.out.println("Hello");
- }
-}
diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude.xoo b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude.xoo
deleted file mode 100644
index 35965a8484c..00000000000
--- a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude.xoo
+++ /dev/null
@@ -1 +0,0 @@
-this file should be excluded from indexing.
diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude2.xoo b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude2.xoo
deleted file mode 100644
index d04e465a561..00000000000
--- a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude2.xoo
+++ /dev/null
@@ -1 +0,0 @@
-this file should ALSO be excluded from indexing.