From 48705fd9366d3cfae6c4642084d0400ee73313ad Mon Sep 17 00:00:00 2001
From: Matteo Mara
Date: Fri, 22 Dec 2023 10:28:20 +0100
Subject: Revert "SONAR-21195 Refactor file indexing into two distinct steps"
This reverts commit 42d16dafde01b19bc7fcb99084e087fdc9b8e95e.
---
.../scanner/mediumtest/ScannerMediumTester.java | 8 +-
.../scanner/mediumtest/fs/FileSystemMediumIT.java | 43 +---
.../sonar/scanner/bootstrap/PluginInstaller.java | 11 +-
.../scanner/bootstrap/ScannerPluginInstaller.java | 11 +-
.../scanner/bootstrap/ScannerPluginRepository.java | 20 +-
.../scanner/bootstrap/SpringScannerContainer.java | 12 +-
.../scanner/mediumtest/FakePluginInstaller.java | 21 +-
.../org/sonar/scanner/mediumtest/LocalPlugin.java | 33 ---
.../scanner/scan/SpringProjectScanContainer.java | 15 +-
.../scan/filesystem/DirectoryFileVisitor.java | 152 --------------
.../sonar/scanner/scan/filesystem/FileIndexer.java | 141 ++++++++++---
.../scanner/scan/filesystem/FilePreprocessor.java | 139 ------------
.../scan/filesystem/InputFileFilterRepository.java | 34 ---
.../scanner/scan/filesystem/LanguageDetection.java | 16 +-
.../scan/filesystem/ModuleRelativePathWarner.java | 47 -----
.../scan/filesystem/ProjectFileIndexer.java | 232 ++++++++++++++++++---
.../scan/filesystem/ProjectFilePreprocessor.java | 230 --------------------
.../scan/filesystem/DirectoryFileVisitorTest.java | 93 ---------
.../filesystem/InputFileFilterRepositoryTest.java | 41 ----
.../scan/filesystem/LanguageDetectionTest.java | 33 +--
.../sonar-project.properties | 5 -
.../xources/hello/HelloJava.xoo | 8 -
.../xources/hello/xoo_exclude.xoo | 1 -
.../xources/hello/xoo_exclude2.xoo | 1 -
24 files changed, 355 insertions(+), 992 deletions(-)
delete mode 100644 sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/LocalPlugin.java
delete mode 100644 sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java
delete mode 100644 sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java
delete mode 100644 sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepository.java
delete mode 100644 sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleRelativePathWarner.java
delete mode 100644 sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java
delete mode 100644 sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitorTest.java
delete mode 100644 sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepositoryTest.java
delete mode 100644 sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/sonar-project.properties
delete mode 100644 sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/HelloJava.xoo
delete mode 100644 sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude.xoo
delete mode 100644 sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude2.xoo
(limited to 'sonar-scanner-engine')
diff --git a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/ScannerMediumTester.java b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/ScannerMediumTester.java
index 1916d3e6d00..ebf0dc20efa 100644
--- a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/ScannerMediumTester.java
+++ b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/ScannerMediumTester.java
@@ -35,7 +35,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
-import java.util.Set;
import javax.annotation.CheckForNull;
import javax.annotation.Nullable;
import javax.annotation.Priority;
@@ -137,12 +136,11 @@ public class ScannerMediumTester extends ExternalResource {
}
public ScannerMediumTester registerPlugin(String pluginKey, Plugin instance) {
- pluginInstaller.add(pluginKey, instance);
- return this;
+ return registerPlugin(pluginKey, instance, 1L);
}
- public ScannerMediumTester registerOptionalPlugin(String pluginKey, Set requiredForLanguages, Plugin instance) {
- pluginInstaller.addOptional(pluginKey, requiredForLanguages, instance);
+ public ScannerMediumTester registerPlugin(String pluginKey, Plugin instance, long lastUpdatedAt) {
+ pluginInstaller.add(pluginKey, instance, lastUpdatedAt);
return this;
}
diff --git a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java
index 66f35f9cfa8..0bfef5a38ad 100644
--- a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java
+++ b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java
@@ -29,7 +29,6 @@ import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Random;
-import java.util.Set;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.SystemUtils;
@@ -40,10 +39,8 @@ import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.slf4j.event.Level;
import org.sonar.api.CoreProperties;
-import org.sonar.api.Plugin;
import org.sonar.api.SonarEdition;
import org.sonar.api.batch.fs.InputFile;
-import org.sonar.api.batch.fs.InputFileFilter;
import org.sonar.api.batch.fs.internal.DefaultInputFile;
import org.sonar.api.testfixtures.log.LogTester;
import org.sonar.api.utils.MessageException;
@@ -77,7 +74,6 @@ public class FileSystemMediumIT {
public ScannerMediumTester tester = new ScannerMediumTester()
.setEdition(SonarEdition.COMMUNITY)
.registerPlugin("xoo", new XooPlugin())
- .registerOptionalPlugin("optional-xoo", Set.of("xoo"), new OptionalXooPlugin())
.addDefaultQProfile("xoo", "Sonar Way")
.addDefaultQProfile("xoo2", "Sonar Way");
@@ -1185,7 +1181,7 @@ public class FileSystemMediumIT {
assertThatThrownBy(result::execute)
.isExactlyInstanceOf(IllegalStateException.class)
- .hasMessageEndingWith(format("Failed to preprocess files"));
+ .hasMessageEndingWith(format("Failed to index files"));
}
@Test
@@ -1256,42 +1252,7 @@ public class FileSystemMediumIT {
assertThatThrownBy(result::execute)
.isExactlyInstanceOf(IllegalStateException.class)
- .hasMessageEndingWith(format("Failed to preprocess files"));
- }
-
- @Test
- public void should_load_input_file_filters_for_required_and_optional_plugins() throws IOException {
- File projectDir = new File("test-resources/mediumtest/xoo/sample-with-input-file-filters");
- AnalysisResult result = tester
- .newAnalysis(new File(projectDir, "sonar-project.properties"))
- .execute();
-
- assertThat(result.inputFiles()).hasSize(1);
-
- assertThat(logTester.logs()).contains("'xources/hello/xoo_exclude2.xoo' excluded by org.sonar.scanner.mediumtest.fs" +
- ".FileSystemMediumIT$OptionalXooPlugin$OptionalXooFileFilter");
- assertThat(logTester.logs()).contains("'xources/hello/xoo_exclude.xoo' excluded by org.sonar.xoo.extensions.XooExcludeFileFilter");
- assertThat(logTester.logs()).contains("'xources/hello/HelloJava.xoo' indexed with language 'xoo'");
-
- assertThat(result.inputFile("xources/hello/xoo_exclude.xoo")).isNull();
- assertThat(result.inputFile("xources/hello/xoo_exclude2.xoo")).isNull();
- assertThat(result.inputFile("xources/hello/HelloJava.xoo")).isNotNull();
- }
-
- public static class OptionalXooPlugin implements Plugin {
-
- @Override
- public void define(Context context) {
- context.addExtension(OptionalXooFileFilter.class);
- }
-
- public static class OptionalXooFileFilter implements InputFileFilter {
-
- @Override
- public boolean accept(InputFile f) {
- return !f.filename().endsWith("_exclude2.xoo");
- }
- }
+ .hasMessageEndingWith(format("Failed to index files"));
}
private static void assertAnalysedFiles(AnalysisResult result, String... files) {
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java
index de69726669c..eeb35462dab 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginInstaller.java
@@ -22,7 +22,6 @@ package org.sonar.scanner.bootstrap;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.sonar.scanner.mediumtest.LocalPlugin;
public interface PluginInstaller {
@@ -46,14 +45,8 @@ public interface PluginInstaller {
Map installPluginsForLanguages(Set languageKeys);
/**
- * Used only by medium tests. Installs required plugins (phase 1)
+ * Used only by medium tests.
* @see org.sonar.scanner.mediumtest.ScannerMediumTester
*/
- List installLocals();
-
- /**
- * Used only by medium tests. Installs optional plugins (phase 2)
- * @see org.sonar.scanner.mediumtest.ScannerMediumTester
- */
- List installOptionalLocals(Set languageKeys);
+ List
- *
- * @param file a reference to the file
- * @param exc the I/O exception that prevented the file from being visited
- * @throws IOException
- */
- @Override
- public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
- if (exc instanceof FileSystemLoopException) {
- LOG.warn("Not indexing due to symlink loop: {}", file.toFile());
- return FileVisitResult.CONTINUE;
- } else if (exc instanceof AccessDeniedException && isExcluded(file)) {
- return FileVisitResult.CONTINUE;
- }
- throw exc;
- }
-
- /**
- * Checks if the directory is excluded in the analysis or not. Only the exclusions are checked.
- *
- * The inclusions cannot be checked for directories, since the current implementation of pattern matching is intended only for files.
- *
- * @param path The file or directory.
- * @return True if file/directory is excluded from the analysis, false otherwise.
- */
- private boolean isExcluded(Path path) throws IOException {
- Path realAbsoluteFile = path.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize();
- return isExcludedDirectory(moduleExclusionFilters, realAbsoluteFile, inputModuleHierarchy.root().getBaseDir(), module.getBaseDir(), type);
- }
-
- /**
- * Checks if the path is a directory that is excluded.
- *
- * Exclusions patterns are checked both at project and module level.
- *
- * @param moduleExclusionFilters The exclusion filters.
- * @param realAbsoluteFile The path to be checked.
- * @param projectBaseDir The project base directory.
- * @param moduleBaseDir The module base directory.
- * @param type The input file type.
- * @return True if path is an excluded directory, false otherwise.
- */
- private static boolean isExcludedDirectory(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectBaseDir, Path moduleBaseDir,
- InputFile.Type type) {
- Path projectRelativePath = projectBaseDir.relativize(realAbsoluteFile);
- Path moduleRelativePath = moduleBaseDir.relativize(realAbsoluteFile);
- return moduleExclusionFilters.isExcludedAsParentDirectoryOfExcludedChildren(realAbsoluteFile, projectRelativePath, projectBaseDir, type)
- || moduleExclusionFilters.isExcludedAsParentDirectoryOfExcludedChildren(realAbsoluteFile, moduleRelativePath, moduleBaseDir, type);
- }
-
- @Override
- public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
- return FileVisitResult.CONTINUE;
- }
-
- private static boolean isHidden(Path path) throws IOException {
- if (SystemUtils.IS_OS_WINDOWS) {
- try {
- DosFileAttributes dosFileAttributes = Files.readAttributes(path, DosFileAttributes.class, LinkOption.NOFOLLOW_LINKS);
- return dosFileAttributes.isHidden();
- } catch (UnsupportedOperationException e) {
- return path.toFile().isHidden();
- }
- } else {
- return Files.isHidden(path);
- }
- }
-
- @FunctionalInterface
- interface FileVisitAction {
- void execute(Path file) throws IOException;
- }
-}
-
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java
index 4d264590716..4d9b8c2a377 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java
@@ -19,10 +19,14 @@
*/
package org.sonar.scanner.scan.filesystem;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.util.function.BooleanSupplier;
+import javax.annotation.Nullable;
+import org.apache.commons.io.FilenameUtils;
import org.sonar.api.CoreProperties;
import org.sonar.api.batch.fs.InputFile;
import org.sonar.api.batch.fs.InputFile.Type;
@@ -32,7 +36,11 @@ import org.sonar.api.batch.fs.internal.DefaultInputFile;
import org.sonar.api.batch.fs.internal.DefaultInputModule;
import org.sonar.api.batch.fs.internal.DefaultInputProject;
import org.sonar.api.batch.fs.internal.SensorStrategy;
+import org.sonar.api.batch.scm.IgnoreCommand;
+import org.sonar.api.notifications.AnalysisWarnings;
import org.sonar.api.utils.MessageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.sonar.scanner.issue.ignore.scanner.IssueExclusionsLoader;
import org.sonar.scanner.repository.language.Language;
import org.sonar.scanner.scan.ScanProperties;
@@ -48,7 +56,10 @@ public class FileIndexer {
private static final Logger LOG = LoggerFactory.getLogger(FileIndexer.class);
+ private final AnalysisWarnings analysisWarnings;
private final ScanProperties properties;
+ private final InputFileFilter[] filters;
+ private final ProjectExclusionFilters projectExclusionFilters;
private final ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions;
private final IssueExclusionsLoader issueExclusionsLoader;
private final MetadataGenerator metadataGenerator;
@@ -60,13 +71,15 @@ public class FileIndexer {
private final StatusDetection statusDetection;
private final ScmChangedFiles scmChangedFiles;
- private final ModuleRelativePathWarner moduleRelativePathWarner;
- private final InputFileFilterRepository inputFileFilterRepository;
+ private boolean warnInclusionsAlreadyLogged;
+ private boolean warnExclusionsAlreadyLogged;
+ private boolean warnCoverageExclusionsAlreadyLogged;
+ private boolean warnDuplicationExclusionsAlreadyLogged;
public FileIndexer(DefaultInputProject project, ScannerComponentIdGenerator scannerComponentIdGenerator, InputComponentStore componentStore,
- ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, IssueExclusionsLoader issueExclusionsLoader,
- MetadataGenerator metadataGenerator, SensorStrategy sensorStrategy, LanguageDetection languageDetection, ScanProperties properties,
- ScmChangedFiles scmChangedFiles, StatusDetection statusDetection, ModuleRelativePathWarner moduleRelativePathWarner, InputFileFilterRepository inputFileFilterRepository) {
+ ProjectExclusionFilters projectExclusionFilters, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, IssueExclusionsLoader issueExclusionsLoader,
+ MetadataGenerator metadataGenerator, SensorStrategy sensorStrategy, LanguageDetection languageDetection, AnalysisWarnings analysisWarnings, ScanProperties properties,
+ InputFileFilter[] filters, ScmChangedFiles scmChangedFiles, StatusDetection statusDetection) {
this.project = project;
this.scannerComponentIdGenerator = scannerComponentIdGenerator;
this.componentStore = componentStore;
@@ -75,23 +88,55 @@ public class FileIndexer {
this.metadataGenerator = metadataGenerator;
this.sensorStrategy = sensorStrategy;
this.langDetection = languageDetection;
+ this.analysisWarnings = analysisWarnings;
this.properties = properties;
+ this.filters = filters;
+ this.projectExclusionFilters = projectExclusionFilters;
this.scmChangedFiles = scmChangedFiles;
this.statusDetection = statusDetection;
- this.moduleRelativePathWarner = moduleRelativePathWarner;
- this.inputFileFilterRepository = inputFileFilterRepository;
}
- void indexFile(DefaultInputModule module, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path sourceFile,
- Type type, ProgressReport progressReport) {
- Path projectRelativePath = project.getBaseDir().relativize(sourceFile);
- Path moduleRelativePath = module.getBaseDir().relativize(sourceFile);
+ void indexFile(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions,
+ Path sourceFile, Type type, ProgressReport progressReport, ProjectFileIndexer.ExclusionCounter exclusionCounter, @Nullable IgnoreCommand ignoreCommand)
+ throws IOException {
+ // get case of real file without resolving link
+ Path realAbsoluteFile = sourceFile.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize();
+ Path projectRelativePath = project.getBaseDir().relativize(realAbsoluteFile);
+ Path moduleRelativePath = module.getBaseDir().relativize(realAbsoluteFile);
+ boolean included = evaluateInclusionsFilters(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type);
+ if (!included) {
+ exclusionCounter.increaseByPatternsCount();
+ return;
+ }
+ boolean excluded = evaluateExclusionsFilters(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type);
+ if (excluded) {
+ exclusionCounter.increaseByPatternsCount();
+ return;
+ }
+ if (!realAbsoluteFile.startsWith(project.getBaseDir())) {
+ LOG.warn("File '{}' is ignored. It is not located in project basedir '{}'.", realAbsoluteFile.toAbsolutePath(), project.getBaseDir());
+ return;
+ }
+ if (!realAbsoluteFile.startsWith(module.getBaseDir())) {
+ LOG.warn("File '{}' is ignored. It is not located in module basedir '{}'.", realAbsoluteFile.toAbsolutePath(), module.getBaseDir());
+ return;
+ }
+
+ if (Files.exists(realAbsoluteFile) && isFileSizeBiggerThanLimit(realAbsoluteFile)) {
+ LOG.warn("File '{}' is bigger than {}MB and as consequence is removed from the analysis scope.", realAbsoluteFile.toAbsolutePath(), properties.fileSizeLimit());
+ return;
+ }
+
+ Language language = langDetection.language(realAbsoluteFile, projectRelativePath);
- // This should be fast; language should be cached from preprocessing step
- Language language = langDetection.language(sourceFile, projectRelativePath);
+ if (ignoreCommand != null && ignoreCommand.isIgnored(realAbsoluteFile)) {
+ LOG.debug("File '{}' is excluded by the scm ignore settings.", realAbsoluteFile);
+ exclusionCounter.increaseByScmCount();
+ return;
+ }
DefaultIndexedFile indexedFile = new DefaultIndexedFile(
- sourceFile,
+ realAbsoluteFile,
project.key(),
projectRelativePath.toString(),
moduleRelativePath.toString(),
@@ -99,7 +144,7 @@ public class FileIndexer {
language != null ? language.key() : null,
scannerComponentIdGenerator.getAsInt(),
sensorStrategy,
- scmChangedFiles.getOldRelativeFilePath(sourceFile)
+ scmChangedFiles.getOldRelativeFilePath(realAbsoluteFile)
);
DefaultInputFile inputFile = new DefaultInputFile(indexedFile, f -> metadataGenerator.setMetadata(module.key(), f, module.getEncoding()),
@@ -114,9 +159,7 @@ public class FileIndexer {
componentStore.put(module.key(), inputFile);
issueExclusionsLoader.addMulticriteriaPatterns(inputFile);
String langStr = inputFile.language() != null ? format("with language '%s'", inputFile.language()) : "with no language";
- if (LOG.isDebugEnabled()) {
- LOG.debug("'{}' indexed {}{}", projectRelativePath, type == Type.TEST ? "as test " : "", langStr);
- }
+ LOG.debug("'{}' indexed {}{}", projectRelativePath, type == Type.TEST ? "as test " : "", langStr);
evaluateCoverageExclusions(moduleCoverageAndDuplicationExclusions, inputFile);
evaluateDuplicationExclusions(moduleCoverageAndDuplicationExclusions, inputFile);
if (properties.preloadFileMetadata()) {
@@ -126,6 +169,42 @@ public class FileIndexer {
progressReport.message(count + " " + pluralizeFiles(count) + " indexed... (last one was " + inputFile.getProjectRelativePath() + ")");
}
+ private boolean evaluateInclusionsFilters(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectRelativePath, Path moduleRelativePath,
+ InputFile.Type type) {
+ if (!Arrays.equals(moduleExclusionFilters.getInclusionsConfig(type), projectExclusionFilters.getInclusionsConfig(type))) {
+ // Module specific configuration
+ return moduleExclusionFilters.isIncluded(realAbsoluteFile, moduleRelativePath, type);
+ }
+ boolean includedByProjectConfiguration = projectExclusionFilters.isIncluded(realAbsoluteFile, projectRelativePath, type);
+ if (includedByProjectConfiguration) {
+ return true;
+ } else if (moduleExclusionFilters.isIncluded(realAbsoluteFile, moduleRelativePath, type)) {
+ warnOnce(
+ type == Type.MAIN ? CoreProperties.PROJECT_INCLUSIONS_PROPERTY : CoreProperties.PROJECT_TEST_INCLUSIONS_PROPERTY,
+ FilenameUtils.normalize(projectRelativePath.toString(), true), () -> warnInclusionsAlreadyLogged, () -> warnInclusionsAlreadyLogged = true);
+ return true;
+ }
+ return false;
+ }
+
+ private boolean evaluateExclusionsFilters(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectRelativePath, Path moduleRelativePath,
+ InputFile.Type type) {
+ if (!Arrays.equals(moduleExclusionFilters.getExclusionsConfig(type), projectExclusionFilters.getExclusionsConfig(type))) {
+ // Module specific configuration
+ return moduleExclusionFilters.isExcluded(realAbsoluteFile, moduleRelativePath, type);
+ }
+ boolean includedByProjectConfiguration = projectExclusionFilters.isExcluded(realAbsoluteFile, projectRelativePath, type);
+ if (includedByProjectConfiguration) {
+ return true;
+ } else if (moduleExclusionFilters.isExcluded(realAbsoluteFile, moduleRelativePath, type)) {
+ warnOnce(
+ type == Type.MAIN ? CoreProperties.PROJECT_EXCLUSIONS_PROPERTY : CoreProperties.PROJECT_TEST_EXCLUSIONS_PROPERTY,
+ FilenameUtils.normalize(projectRelativePath.toString(), true), () -> warnExclusionsAlreadyLogged, () -> warnExclusionsAlreadyLogged = true);
+ return true;
+ }
+ return false;
+ }
+
private void checkIfAlreadyIndexed(DefaultInputFile inputFile) {
if (componentStore.inputFile(inputFile.getProjectRelativePath()) != null) {
throw MessageException.of("File " + inputFile + " can't be indexed twice. Please check that inclusion/exclusion patterns produce "
@@ -150,7 +229,8 @@ public class FileIndexer {
if (excludedByProjectConfiguration) {
return true;
} else if (moduleCoverageAndDuplicationExclusions.isExcludedForCoverage(inputFile)) {
- moduleRelativePathWarner.warnOnce(CoreProperties.PROJECT_COVERAGE_EXCLUSIONS_PROPERTY, inputFile.getProjectRelativePath());
+ warnOnce(CoreProperties.PROJECT_COVERAGE_EXCLUSIONS_PROPERTY, inputFile.getProjectRelativePath(), () -> warnCoverageExclusionsAlreadyLogged,
+ () -> warnCoverageExclusionsAlreadyLogged = true);
return true;
}
return false;
@@ -173,15 +253,26 @@ public class FileIndexer {
if (excludedByProjectConfiguration) {
return true;
} else if (moduleCoverageAndDuplicationExclusions.isExcludedForDuplication(inputFile)) {
- moduleRelativePathWarner.warnOnce(CoreProperties.CPD_EXCLUSIONS, inputFile.getProjectRelativePath());
+ warnOnce(CoreProperties.CPD_EXCLUSIONS, inputFile.getProjectRelativePath(), () -> warnDuplicationExclusionsAlreadyLogged,
+ () -> warnDuplicationExclusionsAlreadyLogged = true);
return true;
}
return false;
}
+ private void warnOnce(String propKey, String filePath, BooleanSupplier alreadyLoggedGetter, Runnable markAsLogged) {
+ if (!alreadyLoggedGetter.getAsBoolean()) {
+ String msg = "Specifying module-relative paths at project level in the property '" + propKey + "' is deprecated. " +
+ "To continue matching files like '" + filePath + "', update this property so that patterns refer to project-relative paths.";
+ LOG.warn(msg);
+ analysisWarnings.addUnique(msg);
+ markAsLogged.run();
+ }
+ }
+
private boolean accept(InputFile indexedFile) {
// InputFileFilter extensions. Might trigger generation of metadata
- for (InputFileFilter filter : inputFileFilterRepository.getInputFileFilters()) {
+ for (InputFileFilter filter : filters) {
if (!filter.accept(indexedFile)) {
LOG.debug("'{}' excluded by {}", indexedFile, filter.getClass().getName());
return false;
@@ -194,5 +285,7 @@ public class FileIndexer {
return count == 1 ? "file" : "files";
}
-
+ private boolean isFileSizeBiggerThanLimit(Path filePath) throws IOException {
+ return Files.size(filePath) > properties.fileSizeLimit() * 1024L * 1024L;
+ }
}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java
deleted file mode 100644
index 6faa87cc42a..00000000000
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2023 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.scanner.scan.filesystem;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.LinkOption;
-import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.Optional;
-import javax.annotation.CheckForNull;
-import org.apache.commons.io.FilenameUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.sonar.api.CoreProperties;
-import org.sonar.api.batch.fs.InputFile;
-import org.sonar.api.batch.fs.internal.DefaultInputModule;
-import org.sonar.api.batch.fs.internal.DefaultInputProject;
-import org.sonar.api.batch.scm.IgnoreCommand;
-import org.sonar.scanner.scan.ScanProperties;
-
-public class FilePreprocessor {
-
- private static final Logger LOG = LoggerFactory.getLogger(FilePreprocessor.class);
-
- private final ModuleRelativePathWarner moduleRelativePathWarner;
- private final DefaultInputProject project;
- private final LanguageDetection languageDetection;
- private final ProjectExclusionFilters projectExclusionFilters;
- private final ScanProperties properties;
-
- public FilePreprocessor(ModuleRelativePathWarner moduleRelativePathWarner, DefaultInputProject project,
- LanguageDetection languageDetection, ProjectExclusionFilters projectExclusionFilters, ScanProperties properties) {
- this.moduleRelativePathWarner = moduleRelativePathWarner;
- this.project = project;
- this.languageDetection = languageDetection;
- this.projectExclusionFilters = projectExclusionFilters;
- this.properties = properties;
- }
-
- public Optional processFile(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, Path sourceFile,
- InputFile.Type type, ProjectFilePreprocessor.ExclusionCounter exclusionCounter, @CheckForNull IgnoreCommand ignoreCommand) throws IOException {
- // get case of real file without resolving link
- Path realAbsoluteFile = sourceFile.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize();
- Path projectRelativePath = project.getBaseDir().relativize(realAbsoluteFile);
- Path moduleRelativePath = module.getBaseDir().relativize(realAbsoluteFile);
- boolean included = isFileIncluded(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type);
- if (!included) {
- exclusionCounter.increaseByPatternsCount();
- return Optional.empty();
- }
- boolean excluded = isFileExcluded(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type);
- if (excluded) {
- exclusionCounter.increaseByPatternsCount();
- return Optional.empty();
- }
-
- if (!realAbsoluteFile.startsWith(project.getBaseDir())) {
- LOG.warn("File '{}' is ignored. It is not located in project basedir '{}'.", realAbsoluteFile.toAbsolutePath(), project.getBaseDir());
- return Optional.empty();
- }
- if (!realAbsoluteFile.startsWith(module.getBaseDir())) {
- LOG.warn("File '{}' is ignored. It is not located in module basedir '{}'.", realAbsoluteFile.toAbsolutePath(), module.getBaseDir());
- return Optional.empty();
- }
-
- if (ignoreCommand != null && ignoreCommand.isIgnored(realAbsoluteFile)) {
- LOG.debug("File '{}' is excluded by the scm ignore settings.", realAbsoluteFile);
- exclusionCounter.increaseByScmCount();
- return Optional.empty();
- }
-
- if (Files.exists(realAbsoluteFile) && isFileSizeBiggerThanLimit(realAbsoluteFile)) {
- LOG.warn("File '{}' is bigger than {}MB and as consequence is removed from the analysis scope.", realAbsoluteFile.toAbsolutePath(), properties.fileSizeLimit());
- return Optional.empty();
- }
-
- languageDetection.language(realAbsoluteFile, projectRelativePath);
-
- return Optional.of(realAbsoluteFile);
- }
-
- private boolean isFileIncluded(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectRelativePath,
- Path moduleRelativePath, InputFile.Type type) {
- if (!Arrays.equals(moduleExclusionFilters.getInclusionsConfig(type), projectExclusionFilters.getInclusionsConfig(type))) {
- return moduleExclusionFilters.isIncluded(realAbsoluteFile, moduleRelativePath, type);
- }
- boolean includedByProjectConfiguration = projectExclusionFilters.isIncluded(realAbsoluteFile, projectRelativePath, type);
- if (includedByProjectConfiguration) {
- return true;
- }
- if (moduleExclusionFilters.isIncluded(realAbsoluteFile, moduleRelativePath, type)) {
- moduleRelativePathWarner.warnOnce(
- type == InputFile.Type.MAIN ? CoreProperties.PROJECT_INCLUSIONS_PROPERTY : CoreProperties.PROJECT_TEST_INCLUSIONS_PROPERTY,
- FilenameUtils.normalize(projectRelativePath.toString(), true));
- return true;
- }
- return false;
- }
-
- private boolean isFileExcluded(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectRelativePath,
- Path moduleRelativePath, InputFile.Type type) {
- if (!Arrays.equals(moduleExclusionFilters.getExclusionsConfig(type), projectExclusionFilters.getExclusionsConfig(type))) {
- return moduleExclusionFilters.isExcluded(realAbsoluteFile, moduleRelativePath, type);
- }
- boolean includedByProjectConfiguration = projectExclusionFilters.isExcluded(realAbsoluteFile, projectRelativePath, type);
- if (includedByProjectConfiguration) {
- return true;
- }
- if (moduleExclusionFilters.isExcluded(realAbsoluteFile, moduleRelativePath, type)) {
- moduleRelativePathWarner.warnOnce(
- type == InputFile.Type.MAIN ? CoreProperties.PROJECT_EXCLUSIONS_PROPERTY : CoreProperties.PROJECT_TEST_EXCLUSIONS_PROPERTY,
- FilenameUtils.normalize(projectRelativePath.toString(), true));
- return true;
- }
- return false;
- }
-
- private boolean isFileSizeBiggerThanLimit(Path filePath) throws IOException {
- return Files.size(filePath) > properties.fileSizeLimit() * 1024L * 1024L;
- }
-}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepository.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepository.java
deleted file mode 100644
index b6e14af85eb..00000000000
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepository.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2023 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.scanner.scan.filesystem;
-
-import org.sonar.api.batch.fs.InputFileFilter;
-
-public class InputFileFilterRepository {
- private final InputFileFilter[] inputFileFilters;
-
- public InputFileFilterRepository(InputFileFilter... inputFileFilters) {
- this.inputFileFilters = inputFileFilters;
- }
-
- public InputFileFilter[] getInputFileFilters() {
- return inputFileFilters;
- }
-}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java
index 24128133824..5ede0250c08 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java
@@ -25,7 +25,6 @@ import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.annotation.CheckForNull;
@@ -54,9 +53,8 @@ public class LanguageDetection {
*/
private final Map patternsByLanguage;
private final List languagesToConsider;
- private final Map languageCacheByPath;
- public LanguageDetection(Configuration settings, LanguagesRepository languages, Map languageCache) {
+ public LanguageDetection(Configuration settings, LanguagesRepository languages) {
Map patternsByLanguageBuilder = new LinkedHashMap<>();
for (Language language : languages.all()) {
String[] filePatterns = settings.getStringArray(getFileLangPatternPropKey(language.key()));
@@ -71,7 +69,6 @@ public class LanguageDetection {
languagesToConsider = List.copyOf(patternsByLanguageBuilder.keySet());
patternsByLanguage = unmodifiableMap(patternsByLanguageBuilder);
- languageCacheByPath = languageCache;
}
private static PathPattern[] getLanguagePatterns(Language language) {
@@ -92,16 +89,11 @@ public class LanguageDetection {
@CheckForNull
Language language(Path absolutePath, Path relativePath) {
- Language detectedLanguage = languageCacheByPath.get(absolutePath.toString());
- if (detectedLanguage != null) {
- return detectedLanguage;
- }
-
+ Language detectedLanguage = null;
for (Language language : languagesToConsider) {
if (isCandidateForLanguage(absolutePath, relativePath, language)) {
if (detectedLanguage == null) {
detectedLanguage = language;
- languageCacheByPath.put(absolutePath.toString(), language);
} else {
// Language was already forced by another pattern
throw MessageException.of(MessageFormat.format("Language of file ''{0}'' can not be decided as the file matches patterns of both {1} and {2}",
@@ -113,10 +105,6 @@ public class LanguageDetection {
return detectedLanguage;
}
- public Set getDetectedLanguages() {
- return languageCacheByPath.values().stream().map(Language::key).collect(Collectors.toSet());
- }
-
private boolean isCandidateForLanguage(Path absolutePath, Path relativePath, Language language) {
PathPattern[] patterns = patternsByLanguage.get(language);
return patterns != null && Arrays.stream(patterns).anyMatch(pattern -> pattern.match(absolutePath, relativePath, false));
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleRelativePathWarner.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleRelativePathWarner.java
deleted file mode 100644
index 766955e438c..00000000000
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleRelativePathWarner.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2023 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.scanner.scan.filesystem;
-
-import java.util.HashSet;
-import java.util.Set;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.sonar.api.notifications.AnalysisWarnings;
-
-public class ModuleRelativePathWarner {
-
- private static final Logger LOG = LoggerFactory.getLogger(ModuleRelativePathWarner.class);
- private final AnalysisWarnings analysisWarnings;
- private final Set previouslyWarnedProps = new HashSet<>();
-
- public ModuleRelativePathWarner(AnalysisWarnings analysisWarnings) {
- this.analysisWarnings = analysisWarnings;
- }
-
- public void warnOnce(String propKey, String filePath) {
- if (!previouslyWarnedProps.contains(propKey)) {
- previouslyWarnedProps.add(propKey);
- String msg = "Specifying module-relative paths at project level in the property '" + propKey + "' is deprecated. " +
- "To continue matching files like '" + filePath + "', update this property so that patterns refer to project-relative paths.";
- LOG.warn(msg);
- analysisWarnings.addUnique(msg);
- }
- }
-}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java
index 5ff7ca1ef17..35d5bf3fdb4 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java
@@ -20,22 +20,33 @@
package org.sonar.scanner.scan.filesystem;
import java.io.IOException;
+import java.nio.file.AccessDeniedException;
+import java.nio.file.FileSystemLoopException;
import java.nio.file.FileVisitOption;
+import java.nio.file.FileVisitResult;
+import java.nio.file.FileVisitor;
import java.nio.file.Files;
+import java.nio.file.LinkOption;
import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.DosFileAttributes;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.commons.lang.SystemUtils;
+import org.sonar.api.batch.fs.InputFile;
import org.sonar.api.batch.fs.InputFile.Type;
import org.sonar.api.batch.fs.internal.DefaultInputModule;
+import org.sonar.api.batch.scm.IgnoreCommand;
import org.sonar.api.notifications.AnalysisWarnings;
import org.sonar.api.scan.filesystem.PathResolver;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.sonar.scanner.bootstrap.GlobalConfiguration;
import org.sonar.scanner.bootstrap.GlobalServerSettings;
import org.sonar.scanner.fs.InputModuleHierarchy;
@@ -43,8 +54,12 @@ import org.sonar.scanner.scan.ModuleConfiguration;
import org.sonar.scanner.scan.ModuleConfigurationProvider;
import org.sonar.scanner.scan.ProjectServerSettings;
import org.sonar.scanner.scan.SonarGlobalPropertiesFilter;
+import org.sonar.scanner.scm.ScmConfiguration;
import org.sonar.scanner.util.ProgressReport;
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+
/**
* Index project input files into {@link InputComponentStore}.
*/
@@ -54,13 +69,15 @@ public class ProjectFileIndexer {
private final ProjectExclusionFilters projectExclusionFilters;
private final SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter;
private final ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions;
+ private final ScmConfiguration scmConfiguration;
private final InputComponentStore componentStore;
private final InputModuleHierarchy inputModuleHierarchy;
private final GlobalConfiguration globalConfig;
private final GlobalServerSettings globalServerSettings;
private final ProjectServerSettings projectServerSettings;
private final FileIndexer fileIndexer;
- private final ProjectFilePreprocessor projectFilePreprocessor;
+ private final IgnoreCommand ignoreCommand;
+ private final boolean useScmExclusion;
private final AnalysisWarnings analysisWarnings;
private ProgressReport progressReport;
@@ -68,8 +85,8 @@ public class ProjectFileIndexer {
public ProjectFileIndexer(InputComponentStore componentStore, ProjectExclusionFilters exclusionFilters,
SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter, InputModuleHierarchy inputModuleHierarchy,
GlobalConfiguration globalConfig, GlobalServerSettings globalServerSettings, ProjectServerSettings projectServerSettings,
- FileIndexer fileIndexer, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions,
- ProjectFilePreprocessor projectFilePreprocessor, AnalysisWarnings analysisWarnings) {
+ FileIndexer fileIndexer, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, ScmConfiguration scmConfiguration,
+ AnalysisWarnings analysisWarnings) {
this.componentStore = componentStore;
this.sonarGlobalPropertiesFilter = sonarGlobalPropertiesFilter;
this.inputModuleHierarchy = inputModuleHierarchy;
@@ -79,8 +96,10 @@ public class ProjectFileIndexer {
this.fileIndexer = fileIndexer;
this.projectExclusionFilters = exclusionFilters;
this.projectCoverageAndDuplicationExclusions = projectCoverageAndDuplicationExclusions;
- this.projectFilePreprocessor = projectFilePreprocessor;
+ this.scmConfiguration = scmConfiguration;
this.analysisWarnings = analysisWarnings;
+ this.ignoreCommand = loadIgnoreCommand();
+ this.useScmExclusion = ignoreCommand != null;
}
public void index() {
@@ -89,22 +108,47 @@ public class ProjectFileIndexer {
LOG.info("Project configuration:");
projectExclusionFilters.log(" ");
projectCoverageAndDuplicationExclusions.log(" ");
+ ExclusionCounter exclusionCounter = new ExclusionCounter();
- indexModulesRecursively(inputModuleHierarchy.root());
+ if (useScmExclusion) {
+ ignoreCommand.init(inputModuleHierarchy.root().getBaseDir().toAbsolutePath());
+ indexModulesRecursively(inputModuleHierarchy.root(), exclusionCounter);
+ ignoreCommand.clean();
+ } else {
+ indexModulesRecursively(inputModuleHierarchy.root(), exclusionCounter);
+ }
int totalIndexed = componentStore.inputFiles().size();
progressReport.stop(totalIndexed + " " + pluralizeFiles(totalIndexed) + " indexed");
+ int excludedFileByPatternCount = exclusionCounter.getByPatternsCount();
+ if (projectExclusionFilters.hasPattern() || excludedFileByPatternCount > 0) {
+ LOG.info("{} {} ignored because of inclusion/exclusion patterns", excludedFileByPatternCount, pluralizeFiles(excludedFileByPatternCount));
+ }
+ int excludedFileByScmCount = exclusionCounter.getByScmCount();
+ if (useScmExclusion) {
+ LOG.info("{} {} ignored because of scm ignore settings", excludedFileByScmCount, pluralizeFiles(excludedFileByScmCount));
+ }
}
- private void indexModulesRecursively(DefaultInputModule module) {
- inputModuleHierarchy.children(module).stream()
- .sorted(Comparator.comparing(DefaultInputModule::key))
- .forEach(this::indexModulesRecursively);
- index(module);
+ private IgnoreCommand loadIgnoreCommand() {
+ try {
+ if (!scmConfiguration.isExclusionDisabled() && scmConfiguration.provider() != null) {
+ return scmConfiguration.provider().ignoreCommand();
+ }
+ } catch (UnsupportedOperationException e) {
+ LOG.debug("File exclusion based on SCM ignore information is not available with this plugin.");
+ }
+
+ return null;
}
- private void index(DefaultInputModule module) {
+ private void indexModulesRecursively(DefaultInputModule module, ExclusionCounter exclusionCounter) {
+ inputModuleHierarchy.children(module).stream().sorted(Comparator.comparing(DefaultInputModule::key)).forEach(m -> indexModulesRecursively(m, exclusionCounter));
+ index(module, exclusionCounter);
+ }
+
+ private void index(DefaultInputModule module, ExclusionCounter exclusionCounter) {
// Emulate creation of module level settings
ModuleConfiguration moduleConfig = new ModuleConfigurationProvider(sonarGlobalPropertiesFilter).provide(globalConfig, module, globalServerSettings, projectServerSettings);
ModuleExclusionFilters moduleExclusionFilters = new ModuleExclusionFilters(moduleConfig, analysisWarnings);
@@ -117,10 +161,13 @@ public class ProjectFileIndexer {
moduleExclusionFilters.log(" ");
moduleCoverageAndDuplicationExclusions.log(" ");
}
- List mainSourceDirsOrFiles = projectFilePreprocessor.getMainSourcesByModule(module);
- indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, mainSourceDirsOrFiles, Type.MAIN);
- projectFilePreprocessor.getTestSourcesByModule(module)
- .ifPresent(tests -> indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, tests, Type.TEST));
+ boolean hasChildModules = !module.definition().getSubProjects().isEmpty();
+ boolean hasTests = module.getTestDirsOrFiles().isPresent();
+ // Default to index basedir when no sources provided
+ List mainSourceDirsOrFiles = module.getSourceDirsOrFiles()
+ .orElseGet(() -> hasChildModules || hasTests ? emptyList() : singletonList(module.getBaseDir().toAbsolutePath()));
+ indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, mainSourceDirsOrFiles, Type.MAIN, exclusionCounter);
+ module.getTestDirsOrFiles().ifPresent(tests -> indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, tests, Type.TEST, exclusionCounter));
}
private static void logPaths(String label, Path baseDir, List paths) {
@@ -129,7 +176,7 @@ public class ProjectFileIndexer {
for (Iterator it = paths.iterator(); it.hasNext(); ) {
Path file = it.next();
Optional relativePathToBaseDir = PathResolver.relativize(baseDir, file);
- if (relativePathToBaseDir.isEmpty()) {
+ if (!relativePathToBaseDir.isPresent()) {
sb.append(file);
} else if (StringUtils.isBlank(relativePathToBaseDir.get())) {
sb.append(".");
@@ -148,14 +195,19 @@ public class ProjectFileIndexer {
}
}
- private void indexFiles(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions,
- List sources, Type type) {
+ private static String pluralizeFiles(int count) {
+ return count == 1 ? "file" : "files";
+ }
+
+ private void indexFiles(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters,
+ ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, List sources, Type type, ExclusionCounter exclusionCounter) {
try {
for (Path dirOrFile : sources) {
if (dirOrFile.toFile().isDirectory()) {
- indexDirectory(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type);
+ indexDirectory(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type, exclusionCounter);
} else {
- fileIndexer.indexFile(module, moduleCoverageAndDuplicationExclusions, dirOrFile, type, progressReport);
+ fileIndexer.indexFile(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type, progressReport, exclusionCounter,
+ ignoreCommand);
}
}
} catch (IOException e) {
@@ -164,17 +216,141 @@ public class ProjectFileIndexer {
}
private void indexDirectory(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters,
- ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions,
- Path dirToIndex, Type type) throws IOException {
+ ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path dirToIndex, Type type, ExclusionCounter exclusionCounter)
+ throws IOException {
Files.walkFileTree(dirToIndex.normalize(), Collections.singleton(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE,
- new DirectoryFileVisitor(file -> fileIndexer.indexFile(module, moduleCoverageAndDuplicationExclusions, file, type, progressReport),
- module, moduleExclusionFilters, inputModuleHierarchy, type));
+ new IndexFileVisitor(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, type, exclusionCounter));
}
- private static String pluralizeFiles(int count) {
- return count == 1 ? "file" : "files";
+
+ /**
+ * Checks if the path is a directory that is excluded.
+ *
+ * Exclusions patterns are checked both at project and module level.
+ *
+ * @param moduleExclusionFilters The exclusion filters.
+ * @param realAbsoluteFile The path to be checked.
+ * @param projectBaseDir The project base directory.
+ * @param moduleBaseDir The module base directory.
+ * @param type The input file type.
+ * @return True if path is an excluded directory, false otherwise.
+ */
+ private static boolean isExcludedDirectory(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectBaseDir, Path moduleBaseDir,
+ InputFile.Type type) {
+ Path projectRelativePath = projectBaseDir.relativize(realAbsoluteFile);
+ Path moduleRelativePath = moduleBaseDir.relativize(realAbsoluteFile);
+ return moduleExclusionFilters.isExcludedAsParentDirectoryOfExcludedChildren(realAbsoluteFile, projectRelativePath, projectBaseDir, type)
+ || moduleExclusionFilters.isExcludedAsParentDirectoryOfExcludedChildren(realAbsoluteFile, moduleRelativePath, moduleBaseDir, type);
+ }
+
+ private class IndexFileVisitor implements FileVisitor {
+ private final DefaultInputModule module;
+ private final ModuleExclusionFilters moduleExclusionFilters;
+ private final ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions;
+ private final Type type;
+ private final ExclusionCounter exclusionCounter;
+
+ IndexFileVisitor(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions,
+ Type type,
+ ExclusionCounter exclusionCounter) {
+ this.module = module;
+ this.moduleExclusionFilters = moduleExclusionFilters;
+ this.moduleCoverageAndDuplicationExclusions = moduleCoverageAndDuplicationExclusions;
+ this.type = type;
+ this.exclusionCounter = exclusionCounter;
+ }
+
+ @Override
+ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+ if (isHidden(dir)) {
+ return FileVisitResult.SKIP_SUBTREE;
+ }
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+ if (!Files.isHidden(file)) {
+ fileIndexer.indexFile(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, file, type, progressReport, exclusionCounter, ignoreCommand);
+ }
+ return FileVisitResult.CONTINUE;
+ }
+
+ /**
+ * Overridden method to handle exceptions while visiting files in the analysis.
+ *
+ *
+ *
+ * - FileSystemLoopException - We show a warning that a symlink loop exists and we skip the file.
+ * - AccessDeniedException for excluded files/directories - We skip the file, as files excluded from the analysis, shouldn't throw access exceptions.
+ *
+ *
+ *
+ * @param file a reference to the file
+ * @param exc the I/O exception that prevented the file from being visited
+ * @throws IOException
+ */
+ @Override
+ public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
+ if (exc instanceof FileSystemLoopException) {
+ LOG.warn("Not indexing due to symlink loop: {}", file.toFile());
+ return FileVisitResult.CONTINUE;
+ } else if (exc instanceof AccessDeniedException && isExcluded(file)) {
+ return FileVisitResult.CONTINUE;
+ }
+ throw exc;
+ }
+
+ /**
+ * Checks if the directory is excluded in the analysis or not. Only the exclusions are checked.
+ *
+ * The inclusions cannot be checked for directories, since the current implementation of pattern matching is intended only for files.
+ *
+ * @param path The file or directory.
+ * @return True if file/directory is excluded from the analysis, false otherwise.
+ */
+ private boolean isExcluded(Path path) throws IOException {
+ Path realAbsoluteFile = path.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize();
+ return isExcludedDirectory(moduleExclusionFilters, realAbsoluteFile, inputModuleHierarchy.root().getBaseDir(), module.getBaseDir(), type);
+ }
+
+ @Override
+ public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
+ return FileVisitResult.CONTINUE;
+ }
+
+ private boolean isHidden(Path path) throws IOException {
+ if (SystemUtils.IS_OS_WINDOWS) {
+ try {
+ DosFileAttributes dosFileAttributes = Files.readAttributes(path, DosFileAttributes.class, LinkOption.NOFOLLOW_LINKS);
+ return dosFileAttributes.isHidden();
+ } catch (UnsupportedOperationException e) {
+ return path.toFile().isHidden();
+ }
+ } else {
+ return Files.isHidden(path);
+ }
+ }
}
+ static class ExclusionCounter {
+ private final AtomicInteger excludedByPatternsCount = new AtomicInteger(0);
+ private final AtomicInteger excludedByScmCount = new AtomicInteger(0);
+
+ public void increaseByPatternsCount() {
+ excludedByPatternsCount.incrementAndGet();
+ }
+
+ public int getByPatternsCount() {
+ return excludedByPatternsCount.get();
+ }
+ public void increaseByScmCount() {
+ excludedByScmCount.incrementAndGet();
+ }
+ public int getByScmCount() {
+ return excludedByScmCount.get();
+ }
+ }
}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java
deleted file mode 100644
index 54d4f2b8d6c..00000000000
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java
+++ /dev/null
@@ -1,230 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2023 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.scanner.scan.filesystem;
-
-import java.io.IOException;
-import java.nio.file.FileVisitOption;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.concurrent.TimeUnit;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.sonar.api.batch.fs.InputFile;
-import org.sonar.api.batch.fs.internal.DefaultInputModule;
-import org.sonar.api.batch.scm.IgnoreCommand;
-import org.sonar.api.batch.scm.ScmProvider;
-import org.sonar.api.notifications.AnalysisWarnings;
-import org.sonar.scanner.bootstrap.GlobalConfiguration;
-import org.sonar.scanner.bootstrap.GlobalServerSettings;
-import org.sonar.scanner.fs.InputModuleHierarchy;
-import org.sonar.scanner.scan.ModuleConfiguration;
-import org.sonar.scanner.scan.ModuleConfigurationProvider;
-import org.sonar.scanner.scan.ProjectServerSettings;
-import org.sonar.scanner.scan.SonarGlobalPropertiesFilter;
-import org.sonar.scanner.scm.ScmConfiguration;
-import org.sonar.scanner.util.ProgressReport;
-
-import static java.util.Collections.emptyList;
-import static java.util.Collections.singletonList;
-
-public class ProjectFilePreprocessor {
-
- private static final Logger LOG = LoggerFactory.getLogger(ProjectFilePreprocessor.class);
- private static final String TELEMETRY_STEP_NAME = "file.preprocessing";
-
- private final AnalysisWarnings analysisWarnings;
- private final IgnoreCommand ignoreCommand;
- private final boolean useScmExclusion;
- private final ScmConfiguration scmConfiguration;
- private final InputModuleHierarchy inputModuleHierarchy;
- private final GlobalConfiguration globalConfig;
- private final GlobalServerSettings globalServerSettings;
- private final ProjectServerSettings projectServerSettings;
- private final LanguageDetection languageDetection;
- private final FilePreprocessor filePreprocessor;
- private final ProjectExclusionFilters projectExclusionFilters;
-
- private final SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter;
-
- private final Map> mainSourcesByModule = new HashMap<>();
- private final Map> testSourcesByModule = new HashMap<>();
-
- private int totalFilesPreprocessed = 0;
-
- public ProjectFilePreprocessor(AnalysisWarnings analysisWarnings, ScmConfiguration scmConfiguration, InputModuleHierarchy inputModuleHierarchy,
- GlobalConfiguration globalConfig, GlobalServerSettings globalServerSettings, ProjectServerSettings projectServerSettings,
- LanguageDetection languageDetection, FilePreprocessor filePreprocessor,
- ProjectExclusionFilters projectExclusionFilters, SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter) {
- this.analysisWarnings = analysisWarnings;
- this.scmConfiguration = scmConfiguration;
- this.inputModuleHierarchy = inputModuleHierarchy;
- this.globalConfig = globalConfig;
- this.globalServerSettings = globalServerSettings;
- this.projectServerSettings = projectServerSettings;
- this.languageDetection = languageDetection;
- this.filePreprocessor = filePreprocessor;
- this.projectExclusionFilters = projectExclusionFilters;
- this.sonarGlobalPropertiesFilter = sonarGlobalPropertiesFilter;
- this.ignoreCommand = loadIgnoreCommand();
- this.useScmExclusion = ignoreCommand != null;
- }
-
- public void execute() {
- ProgressReport progressReport = new ProgressReport("Report about progress of file preprocessing",
- TimeUnit.SECONDS.toMillis(10));
- progressReport.start("Preprocessing files...");
- ExclusionCounter exclusionCounter = new ExclusionCounter();
-
- if (useScmExclusion) {
- ignoreCommand.init(inputModuleHierarchy.root().getBaseDir().toAbsolutePath());
- processModulesRecursively(inputModuleHierarchy.root(), exclusionCounter);
- ignoreCommand.clean();
- } else {
- processModulesRecursively(inputModuleHierarchy.root(), exclusionCounter);
- }
-
- int totalLanguagesDetected = languageDetection.getDetectedLanguages().size();
-
- progressReport.stop(String.format("%s detected in %s", pluralizeWithCount("language", totalLanguagesDetected),
- pluralizeWithCount("preprocessed file", totalFilesPreprocessed)));
-
- int excludedFileByPatternCount = exclusionCounter.getByPatternsCount();
- if (projectExclusionFilters.hasPattern() || excludedFileByPatternCount > 0) {
- if (LOG.isInfoEnabled()) {
- LOG.info("{} ignored because of inclusion/exclusion patterns", pluralizeWithCount("file", excludedFileByPatternCount));
- }
- }
-
- int excludedFileByScmCount = exclusionCounter.getByScmCount();
- if (useScmExclusion) {
- if (LOG.isInfoEnabled()) {
- LOG.info("{} ignored because of scm ignore settings", pluralizeWithCount("file", excludedFileByScmCount));
- }
- }
- }
-
- private void processModulesRecursively(DefaultInputModule module, ExclusionCounter exclusionCounter) {
- inputModuleHierarchy.children(module).stream().sorted(Comparator.comparing(DefaultInputModule::key)).forEach(
- m -> processModulesRecursively(m, exclusionCounter));
- processModule(module, exclusionCounter);
- }
-
- private void processModule(DefaultInputModule module, ExclusionCounter exclusionCounter) {
- // Emulate creation of module level settings
- ModuleConfiguration moduleConfig = new ModuleConfigurationProvider(sonarGlobalPropertiesFilter).provide(globalConfig, module, globalServerSettings, projectServerSettings);
- ModuleExclusionFilters moduleExclusionFilters = new ModuleExclusionFilters(moduleConfig, analysisWarnings);
- boolean hasChildModules = !module.definition().getSubProjects().isEmpty();
- boolean hasTests = module.getTestDirsOrFiles().isPresent();
- // Default to index basedir when no sources provided
- List mainSourceDirsOrFiles = module.getSourceDirsOrFiles()
- .orElseGet(() -> hasChildModules || hasTests ? emptyList() : singletonList(module.getBaseDir().toAbsolutePath()));
- List processedSources = processModuleSources(module, moduleExclusionFilters, mainSourceDirsOrFiles, InputFile.Type.MAIN,
- exclusionCounter);
- mainSourcesByModule.put(module, processedSources);
- totalFilesPreprocessed += processedSources.size();
- module.getTestDirsOrFiles().ifPresent(tests -> {
- List processedTestSources = processModuleSources(module, moduleExclusionFilters, tests, InputFile.Type.TEST, exclusionCounter);
- testSourcesByModule.put(module, processedTestSources);
- totalFilesPreprocessed += processedTestSources.size();
- });
- }
-
- private List processModuleSources(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, List sources,
- InputFile.Type type, ExclusionCounter exclusionCounter) {
- List processedFiles = new ArrayList<>();
- try {
- for (Path dirOrFile : sources) {
- if (dirOrFile.toFile().isDirectory()) {
- processedFiles.addAll(processDirectory(module, moduleExclusionFilters, dirOrFile, type, exclusionCounter));
- } else {
- filePreprocessor.processFile(module, moduleExclusionFilters, dirOrFile, type, exclusionCounter, ignoreCommand)
- .ifPresent(processedFiles::add);
- }
- }
- } catch (IOException e) {
- throw new IllegalStateException("Failed to preprocess files", e);
- }
- return processedFiles;
- }
-
- private List processDirectory(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, Path path,
- InputFile.Type type, ExclusionCounter exclusionCounter) throws IOException {
- List processedFiles = new ArrayList<>();
- Files.walkFileTree(path.normalize(), Collections.singleton(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE,
- new DirectoryFileVisitor(file -> filePreprocessor.processFile(module, moduleExclusionFilters, file, type, exclusionCounter,
- ignoreCommand).ifPresent(processedFiles::add), module, moduleExclusionFilters, inputModuleHierarchy, type)
- );
- return processedFiles;
- }
-
- public List getMainSourcesByModule(DefaultInputModule module) {
- return Collections.unmodifiableList(mainSourcesByModule.get(module));
- }
-
- public Optional> getTestSourcesByModule(DefaultInputModule module) {
- return Optional.ofNullable(testSourcesByModule.get(module)).map(Collections::unmodifiableList);
- }
-
- private IgnoreCommand loadIgnoreCommand() {
- try {
- ScmProvider provider = scmConfiguration.provider();
- if (!scmConfiguration.isExclusionDisabled() && provider != null) {
- return provider.ignoreCommand();
- }
- } catch (UnsupportedOperationException e) {
- LOG.debug("File exclusion based on SCM ignore information is not available with this plugin.");
- }
-
- return null;
- }
-
- private static String pluralizeWithCount(String str, int count) {
- String pluralized = count == 1 ? str : (str + "s");
- return count + " " + pluralized;
- }
-
- public static class ExclusionCounter {
- private int excludedByPatternsCount = 0;
- private int excludedByScmCount = 0;
-
- public void increaseByPatternsCount() {
- excludedByPatternsCount++;
- }
-
- public int getByPatternsCount() {
- return excludedByPatternsCount;
- }
-
- public void increaseByScmCount() {
- excludedByScmCount++;
- }
-
- public int getByScmCount() {
- return excludedByScmCount;
- }
- }
-}
diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitorTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitorTest.java
deleted file mode 100644
index fd79238f65d..00000000000
--- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitorTest.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2023 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.scanner.scan.filesystem;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.FileSystemLoopException;
-import java.nio.file.FileVisitResult;
-import java.nio.file.Files;
-import java.nio.file.LinkOption;
-import java.nio.file.Path;
-import java.nio.file.attribute.BasicFileAttributes;
-import org.apache.commons.lang.SystemUtils;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.sonar.api.batch.fs.InputFile;
-import org.sonar.api.batch.fs.internal.DefaultInputModule;
-import org.sonar.scanner.fs.InputModuleHierarchy;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.assertThrows;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.verify;
-
-public class DirectoryFileVisitorTest {
-
- @ClassRule
- public static TemporaryFolder temp = new TemporaryFolder();
-
- private final DefaultInputModule module = mock();
- private final ModuleExclusionFilters moduleExclusionFilters = mock();
- private final InputModuleHierarchy inputModuleHierarchy = mock();
- private final InputFile.Type type = mock();
-
- @Test
- public void visit_hidden_file() throws IOException {
- DirectoryFileVisitor.FileVisitAction action = mock(DirectoryFileVisitor.FileVisitAction.class);
-
- File hidden = temp.newFile(".hidden");
- if (SystemUtils.IS_OS_WINDOWS) {
- Files.setAttribute(hidden.toPath(), "dos:hidden", true, LinkOption.NOFOLLOW_LINKS);
- }
-
-
- DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleExclusionFilters, inputModuleHierarchy, type);
- underTest.visitFile(hidden.toPath(), Files.readAttributes(hidden.toPath(), BasicFileAttributes.class));
-
- verify(action, never()).execute(any(Path.class));
- }
-
- @Test
- public void test_visit_file_failed_generic_io_exception() throws IOException {
- DirectoryFileVisitor.FileVisitAction action = mock(DirectoryFileVisitor.FileVisitAction.class);
-
- File file = temp.newFile("failed");
-
- DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleExclusionFilters, inputModuleHierarchy, type);
- assertThrows(IOException.class, () -> underTest.visitFileFailed(file.toPath(), new IOException()));
- }
-
- @Test
- public void test_visit_file_failed_file_system_loop_exception() throws IOException {
- DirectoryFileVisitor.FileVisitAction action = mock(DirectoryFileVisitor.FileVisitAction.class);
-
- File file = temp.newFile("symlink");
-
- DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleExclusionFilters, inputModuleHierarchy, type);
- FileVisitResult result = underTest.visitFileFailed(file.toPath(), new FileSystemLoopException(file.getPath()));
-
- assertThat(result).isEqualTo(FileVisitResult.CONTINUE);
- }
-
-}
diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepositoryTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepositoryTest.java
deleted file mode 100644
index 12f1cb0c63a..00000000000
--- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepositoryTest.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2023 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.scanner.scan.filesystem;
-
-import org.junit.Test;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class InputFileFilterRepositoryTest {
-
- @Test
- public void should_not_return_null_if_initialized_with_no_filters() {
- InputFileFilterRepository underTest = new InputFileFilterRepository();
- assertThat(underTest.getInputFileFilters()).isNotNull();
- }
-
- @Test
- public void should_return_filters_from_initialization() {
- InputFileFilterRepository underTest = new InputFileFilterRepository(f -> true);
- assertThat(underTest.getInputFileFilters()).isNotNull();
- assertThat(underTest.getInputFileFilters()).hasSize(1);
- }
-
-}
diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java
index 18408451557..e5810baba5a 100644
--- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java
+++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java
@@ -24,8 +24,6 @@ import com.tngtech.java.junit.dataprovider.DataProviderRunner;
import com.tngtech.java.junit.dataprovider.UseDataProvider;
import java.io.File;
import java.nio.file.Paths;
-import java.util.HashMap;
-import java.util.Map;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -40,11 +38,7 @@ import org.sonar.scanner.repository.language.LanguagesRepository;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.endsWith;
import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
@RunWith(DataProviderRunner.class)
public class LanguageDetectionTest {
@@ -78,7 +72,7 @@ public class LanguageDetectionTest {
@Test
public void detectLanguageKey_shouldDetectByFileExtension() {
LanguagesRepository languages = new FakeLanguagesRepository(new Languages(new MockLanguage("java", "java", "jav"), new MockLanguage("cobol", "cbl", "cob")));
- LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>());
+ LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
assertThat(detectLanguageKey(detection, "Foo.java")).isEqualTo("java");
assertThat(detectLanguageKey(detection, "src/Foo.java")).isEqualTo("java");
@@ -100,7 +94,7 @@ public class LanguageDetectionTest {
new MockLanguage("docker", new String[0], new String[] {"*.dockerfile", "*.Dockerfile", "Dockerfile", "Dockerfile.*"}),
new MockLanguage("terraform", new String[] {"tf"}, new String[] {".tf"}),
new MockLanguage("java", new String[0], new String[] {"**/*Test.java"})));
- LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>());
+ LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
assertThat(detectLanguageKey(detection, fileName)).isEqualTo(expectedLanguageKey);
}
@@ -123,7 +117,7 @@ public class LanguageDetectionTest {
@Test
public void detectLanguageKey_shouldNotFailIfNoLanguage() {
- LanguageDetection detection = spy(new LanguageDetection(settings.asConfig(), new FakeLanguagesRepository(new Languages()), new HashMap<>()));
+ LanguageDetection detection = spy(new LanguageDetection(settings.asConfig(), new FakeLanguagesRepository(new Languages())));
assertThat(detectLanguageKey(detection, "Foo.java")).isNull();
}
@@ -131,14 +125,14 @@ public class LanguageDetectionTest {
public void detectLanguageKey_shouldAllowPluginsToDeclareFileExtensionTwiceForCaseSensitivity() {
LanguagesRepository languages = new FakeLanguagesRepository(new Languages(new MockLanguage("abap", "abap", "ABAP")));
- LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>());
+ LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
assertThat(detectLanguageKey(detection, "abc.abap")).isEqualTo("abap");
}
@Test
public void detectLanguageKey_shouldFailIfConflictingLanguageSuffix() {
LanguagesRepository languages = new FakeLanguagesRepository(new Languages(new MockLanguage("xml", "xhtml"), new MockLanguage("web", "xhtml")));
- LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>());
+ LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
assertThatThrownBy(() -> detectLanguageKey(detection, "abc.xhtml"))
.isInstanceOf(MessageException.class)
.hasMessageContaining("Language of file 'abc.xhtml' can not be decided as the file matches patterns of both ")
@@ -152,7 +146,7 @@ public class LanguageDetectionTest {
settings.setProperty("sonar.lang.patterns.xml", "xml/**");
settings.setProperty("sonar.lang.patterns.web", "web/**");
- LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>());
+ LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
assertThat(detectLanguageKey(detection, "xml/abc.xhtml")).isEqualTo("xml");
assertThat(detectLanguageKey(detection, "web/abc.xhtml")).isEqualTo("web");
}
@@ -163,7 +157,7 @@ public class LanguageDetectionTest {
settings.setProperty("sonar.lang.patterns.abap", "*.abap,*.txt");
settings.setProperty("sonar.lang.patterns.cobol", "*.cobol,*.txt");
- LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>());
+ LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
assertThat(detectLanguageKey(detection, "abc.abap")).isEqualTo("abap");
assertThat(detectLanguageKey(detection, "abc.cobol")).isEqualTo("cobol");
@@ -174,19 +168,6 @@ public class LanguageDetectionTest {
.hasMessageContaining("sonar.lang.patterns.cobol : *.cobol,*.txt");
}
- @Test
- public void should_cache_detected_language_by_file_path() {
- Map languageCacheSpy = spy(new HashMap<>());
- LanguagesRepository languages = new FakeLanguagesRepository(new Languages(
- new MockLanguage("java", "java", "jav"), new MockLanguage("cobol", "cbl", "cob")));
- LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, languageCacheSpy);
-
- assertThat(detectLanguageKey(detection, "Foo.java")).isEqualTo("java");
- assertThat(detectLanguageKey(detection, "Foo.java")).isEqualTo("java");
- verify(languageCacheSpy, times(1)).put(endsWith("/Foo.java"), any(org.sonar.scanner.repository.language.Language.class));
- verify(languageCacheSpy, times(2)).get(endsWith("/Foo.java"));
- }
-
private String detectLanguageKey(LanguageDetection detection, String path) {
org.sonar.scanner.repository.language.Language language = detection.language(new File(temp.getRoot(), path).toPath(), Paths.get(path));
return language != null ? language.key() : null;
diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/sonar-project.properties b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/sonar-project.properties
deleted file mode 100644
index 57c2c062b0e..00000000000
--- a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/sonar-project.properties
+++ /dev/null
@@ -1,5 +0,0 @@
-sonar.organization=org1
-sonar.projectKey=sample-with-empty-file
-sonar.projectName=Sample With Empty
-sonar.projectVersion=0.1-SNAPSHOT
-sonar.sources=xources
diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/HelloJava.xoo b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/HelloJava.xoo
deleted file mode 100644
index ee9bf789a53..00000000000
--- a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/HelloJava.xoo
+++ /dev/null
@@ -1,8 +0,0 @@
-package hello;
-
-public class HelloJava {
-
- public static void main(String[] args) {
- System.out.println("Hello");
- }
-}
diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude.xoo b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude.xoo
deleted file mode 100644
index 35965a8484c..00000000000
--- a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude.xoo
+++ /dev/null
@@ -1 +0,0 @@
-this file should be excluded from indexing.
diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude2.xoo b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude2.xoo
deleted file mode 100644
index d04e465a561..00000000000
--- a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-input-file-filters/xources/hello/xoo_exclude2.xoo
+++ /dev/null
@@ -1 +0,0 @@
-this file should ALSO be excluded from indexing.
--
cgit v1.2.3