diff options
author | Julien HENRY <julien.henry@sonarsource.com> | 2017-07-31 18:44:48 +0200 |
---|---|---|
committer | Julien HENRY <julien.henry@sonarsource.com> | 2017-08-04 14:58:16 +0200 |
commit | 25aaeb2880043aeb8c219bcc2d178041e76062b1 (patch) | |
tree | d99c993fd7657d0dd10023d1447e84f6c77ff185 /sonar-scanner-engine | |
parent | 34b0fd70f5c0e5b4298cba0b2fefe4a1688f7622 (diff) | |
download | sonarqube-25aaeb2880043aeb8c219bcc2d178041e76062b1.tar.gz sonarqube-25aaeb2880043aeb8c219bcc2d178041e76062b1.zip |
SONAR-9561 File inclusions/exclusions should be applied before language detection
Diffstat (limited to 'sonar-scanner-engine')
8 files changed, 109 insertions, 70 deletions
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ExclusionFilters.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ExclusionFilters.java index f84bec040cf..044338518bc 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ExclusionFilters.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ExclusionFilters.java @@ -19,11 +19,11 @@ */ package org.sonar.scanner.scan.filesystem; +import java.nio.file.Path; import org.apache.commons.lang.ArrayUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.api.batch.ScannerSide; -import org.sonar.api.batch.fs.IndexedFile; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.PathPattern; import org.sonar.api.scan.filesystem.FileExclusions; @@ -68,7 +68,7 @@ public class ExclusionFilters { } } - public boolean accept(IndexedFile indexedFile, InputFile.Type type) { + public boolean accept(Path absolutePath, Path relativePath, InputFile.Type type) { PathPattern[] inclusionPatterns; PathPattern[] exclusionPatterns; if (InputFile.Type.MAIN == type) { @@ -84,7 +84,7 @@ public class ExclusionFilters { if (inclusionPatterns.length > 0) { boolean matchInclusion = false; for (PathPattern pattern : inclusionPatterns) { - matchInclusion |= pattern.match(indexedFile.absolutePath(), indexedFile.relativePath()); + matchInclusion |= pattern.match(absolutePath, relativePath); } if (!matchInclusion) { return false; @@ -92,7 +92,7 @@ public class ExclusionFilters { } if (exclusionPatterns.length > 0) { for (PathPattern pattern : exclusionPatterns) { - if (pattern.match(indexedFile.absolutePath(), indexedFile.relativePath())) { + if (pattern.match(absolutePath, relativePath)) { return false; } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java index 6e00933fbd5..a1734423701 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java @@ -73,11 +73,13 @@ public class FileIndexer { private ExecutorService executorService; private final List<Future<Void>> tasks; private final DefaultModuleFileSystem defaultModuleFileSystem; + private final LanguageDetection langDetection; private ProgressReport progressReport; public FileIndexer(BatchIdGenerator batchIdGenerator, InputComponentStore componentStore, DefaultInputModule module, ExclusionFilters exclusionFilters, DefaultComponentTree componentTree, InputFileBuilder inputFileBuilder, ModuleFileSystemInitializer initializer, DefaultModuleFileSystem defaultModuleFileSystem, + LanguageDetection languageDetection, InputFileFilter[] filters) { this.batchIdGenerator = batchIdGenerator; this.componentStore = componentStore; @@ -86,14 +88,17 @@ public class FileIndexer { this.inputFileBuilder = inputFileBuilder; this.moduleFileSystemInitializer = initializer; this.defaultModuleFileSystem = defaultModuleFileSystem; + this.langDetection = languageDetection; this.filters = filters; this.exclusionFilters = exclusionFilters; this.tasks = new ArrayList<>(); } public FileIndexer(BatchIdGenerator batchIdGenerator, InputComponentStore componentStore, DefaultInputModule module, ExclusionFilters exclusionFilters, - DefaultComponentTree componentTree, InputFileBuilder inputFileBuilder, ModuleFileSystemInitializer initializer, DefaultModuleFileSystem defaultModuleFileSystem) { - this(batchIdGenerator, componentStore, module, exclusionFilters, componentTree, inputFileBuilder, initializer, defaultModuleFileSystem, new InputFileFilter[0]); + DefaultComponentTree componentTree, InputFileBuilder inputFileBuilder, ModuleFileSystemInitializer initializer, DefaultModuleFileSystem defaultModuleFileSystem, + LanguageDetection languageDetection) { + this(batchIdGenerator, componentStore, module, exclusionFilters, componentTree, inputFileBuilder, initializer, defaultModuleFileSystem, languageDetection, + new InputFileFilter[0]); } public void index() { @@ -157,21 +162,34 @@ public class FileIndexer { private Void indexFile(Path sourceFile, InputFile.Type type, Progress progress) throws IOException { // get case of real file without resolving link - Path realFile = sourceFile.toRealPath(LinkOption.NOFOLLOW_LINKS); - DefaultInputFile inputFile = inputFileBuilder.create(realFile, type); - if (inputFile != null) { - if (exclusionFilters.accept(inputFile, type) && accept(inputFile)) { - String parentRelativePath = getParentRelativePath(inputFile); - synchronized (this) { - indexParentDir(inputFile, parentRelativePath); - progress.markAsIndexed(inputFile); - } - LOG.debug("'{}' indexed {}with language '{}'", inputFile.relativePath(), type == Type.TEST ? "as test " : "", inputFile.language()); - inputFileBuilder.checkMetadata(inputFile); - } else { - progress.increaseExcludedByPatternsCount(); - } + Path realAbsoluteFile = sourceFile.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize(); + String relativePathStr = PathResolver.relativePath(module.getBaseDir(), realAbsoluteFile); + if (relativePathStr == null) { + LOG.warn("File '{}' is ignored. It is not located in module basedir '{}'.", realAbsoluteFile.toAbsolutePath(), module.getBaseDir()); + return null; + } + Path relativePath = module.getBaseDir().relativize(realAbsoluteFile); + if (!exclusionFilters.accept(realAbsoluteFile, relativePath, type)) { + progress.increaseExcludedByPatternsCount(); + return null; + } + String language = langDetection.language(realAbsoluteFile, relativePath); + if (language == null && langDetection.forcedLanguage() != null) { + LOG.warn("File '{}' is ignored because it doesn't belong to the forced language '{}'", realAbsoluteFile.toAbsolutePath(), langDetection.forcedLanguage()); + return null; + } + DefaultInputFile inputFile = inputFileBuilder.create(type, relativePathStr, language); + if (!accept(inputFile)) { + progress.increaseExcludedByPatternsCount(); + return null; + } + String parentRelativePath = getParentRelativePath(inputFile); + synchronized (this) { + indexParentDir(inputFile, parentRelativePath); + progress.markAsIndexed(inputFile); } + LOG.debug("'{}' indexed {}with language '{}'", relativePathStr, type == Type.TEST ? "as test " : "", inputFile.language()); + inputFileBuilder.checkMetadata(inputFile); return null; } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileBuilder.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileBuilder.java index 1248407e2b1..d8980917519 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileBuilder.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileBuilder.java @@ -20,51 +20,33 @@ package org.sonar.scanner.scan.filesystem; import java.nio.file.Path; -import javax.annotation.CheckForNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DefaultIndexedFile; import org.sonar.api.batch.fs.internal.DefaultInputFile; import org.sonar.api.batch.fs.internal.DefaultInputModule; import org.sonar.api.config.Configuration; -import org.sonar.api.scan.filesystem.PathResolver; public class InputFileBuilder { public static final String PRELOAD_FILE_METADATA_KEY = "sonar.preloadFileMetadata"; - private static final Logger LOG = LoggerFactory.getLogger(InputFileBuilder.class); private final String moduleKey; private final Path moduleBaseDir; - private final LanguageDetection langDetection; private final BatchIdGenerator idGenerator; private final MetadataGenerator metadataGenerator; private final boolean preloadMetadata; private final ModuleFileSystemInitializer moduleFileSystemInitializer; - public InputFileBuilder(DefaultInputModule module, LanguageDetection langDetection, MetadataGenerator metadataGenerator, + public InputFileBuilder(DefaultInputModule module, MetadataGenerator metadataGenerator, BatchIdGenerator idGenerator, Configuration settings, ModuleFileSystemInitializer moduleFileSystemInitializer) { this.moduleFileSystemInitializer = moduleFileSystemInitializer; this.moduleKey = module.key(); this.moduleBaseDir = module.getBaseDir(); - this.langDetection = langDetection; this.metadataGenerator = metadataGenerator; this.idGenerator = idGenerator; this.preloadMetadata = settings.getBoolean(PRELOAD_FILE_METADATA_KEY).orElse(false); } - @CheckForNull - DefaultInputFile create(Path file, InputFile.Type type) { - String relativePath = PathResolver.relativePath(moduleBaseDir, file); - if (relativePath == null) { - LOG.warn("File '{}' is ignored. It is not located in module basedir '{}'.", file.toAbsolutePath(), moduleBaseDir); - return null; - } - String language = langDetection.language(file.toAbsolutePath().normalize().toString(), relativePath); - if (language == null && langDetection.forcedLanguage() != null) { - LOG.warn("File '{}' is ignored because it doesn't belong to the forced language '{}'", file.toAbsolutePath(), langDetection.forcedLanguage()); - return null; - } - + DefaultInputFile create(InputFile.Type type, String relativePath, @Nullable String language) { DefaultIndexedFile indexedFile = new DefaultIndexedFile(moduleKey, moduleBaseDir, relativePath, type, language, idGenerator.get()); DefaultInputFile inputFile = new DefaultInputFile(indexedFile, f -> metadataGenerator.setMetadata(f, moduleFileSystemInitializer.defaultEncoding())); if (language != null) { diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java index 8267e4509c9..7b885a0b428 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java @@ -19,16 +19,16 @@ */ package org.sonar.scanner.scan.filesystem; +import com.google.common.base.Joiner; +import java.nio.file.Path; import java.text.MessageFormat; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; - import javax.annotation.CheckForNull; import javax.annotation.concurrent.ThreadSafe; - import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,8 +40,6 @@ import org.sonar.api.utils.MessageException; import org.sonar.scanner.repository.language.Language; import org.sonar.scanner.repository.language.LanguagesRepository; -import com.google.common.base.Joiner; - /** * Detect language of a source file based on its suffix and configured patterns. */ @@ -103,7 +101,7 @@ public class LanguageDetection { } @CheckForNull - String language(String absolutePath, String relativePath) { + String language(Path absolutePath, Path relativePath) { String detectedLanguage = null; for (String languageKey : languagesToConsider) { if (isCandidateForLanguage(absolutePath, relativePath, languageKey)) { @@ -128,7 +126,7 @@ public class LanguageDetection { return null; } - private boolean isCandidateForLanguage(String absolutePath, String relativePath, String languageKey) { + private boolean isCandidateForLanguage(Path absolutePath, Path relativePath, String languageKey) { PathPattern[] patterns = patternsByLanguage.get(languageKey); if (patterns != null) { for (PathPattern pathPattern : patterns) { diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumTest.java index 016e8c8e554..a61330e3657 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumTest.java @@ -61,6 +61,7 @@ public class FileSystemMediumTest { public ScannerMediumTester tester = new ScannerMediumTester() .registerPlugin("xoo", new XooPlugin()) .addDefaultQProfile("xoo", "Sonar Way") + .addDefaultQProfile("xoo2", "Sonar Way") .setLogOutput(logs); private File baseDir; @@ -670,4 +671,45 @@ public class FileSystemMediumTest { assertThat(result.inputDirs()).hasSize(3); } + @Test + public void twoLanguagesWithSameExtension() throws IOException { + File srcDir = new File(baseDir, "src"); + srcDir.mkdir(); + + File xooFile = new File(srcDir, "sample.xoo"); + FileUtils.write(xooFile, "Sample xoo\ncontent"); + + File xooFile2 = new File(srcDir, "sample.xoo2"); + FileUtils.write(xooFile2, "Sample xoo 2\ncontent"); + + TaskResult result = tester.newTask() + .properties(builder + .put("sonar.sources", "src") + .build()) + .execute(); + + assertThat(result.inputFiles()).hasSize(2); + + try { + result = tester.newTask() + .properties(builder + .put("sonar.lang.patterns.xoo2", "**/*.xoo") + .build()) + .execute(); + } catch (Exception e) { + assertThat(e) + .isInstanceOf(MessageException.class) + .hasMessage( + "Language of file 'src/sample.xoo' can not be decided as the file matches patterns of both sonar.lang.patterns.xoo : **/*.xoo and sonar.lang.patterns.xoo2 : **/*.xoo"); + } + + // SONAR-9561 + result = tester.newTask() + .properties(builder + .put("sonar.exclusions", "**/sample.xoo") + .build()) + .execute(); + + assertThat(result.inputFiles()).hasSize(1); + } } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/ExclusionFiltersTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/ExclusionFiltersTest.java index f64a6729750..186f40189ee 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/ExclusionFiltersTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/ExclusionFiltersTest.java @@ -19,12 +19,10 @@ */ package org.sonar.scanner.scan.filesystem; -import static org.assertj.core.api.Assertions.assertThat; - import java.io.File; import java.io.IOException; import java.nio.file.Path; - +import java.nio.file.Paths; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -36,6 +34,8 @@ import org.sonar.api.batch.fs.internal.DefaultIndexedFile; import org.sonar.api.config.internal.MapSettings; import org.sonar.api.scan.filesystem.FileExclusions; +import static org.assertj.core.api.Assertions.assertThat; + public class ExclusionFiltersTest { @Rule @@ -56,8 +56,8 @@ public class ExclusionFiltersTest { filter.prepare(); IndexedFile indexedFile = new DefaultIndexedFile("foo", moduleBaseDir, "src/main/java/com/mycompany/FooDao.java", null); - assertThat(filter.accept(indexedFile, InputFile.Type.MAIN)).isTrue(); - assertThat(filter.accept(indexedFile, InputFile.Type.TEST)).isTrue(); + assertThat(filter.accept(indexedFile.path(), Paths.get(indexedFile.relativePath()), InputFile.Type.MAIN)).isTrue(); + assertThat(filter.accept(indexedFile.path(), Paths.get(indexedFile.relativePath()), InputFile.Type.TEST)).isTrue(); } @Test @@ -66,10 +66,10 @@ public class ExclusionFiltersTest { filter.prepare(); IndexedFile indexedFile = new DefaultIndexedFile("foo", moduleBaseDir, "src/main/java/com/mycompany/FooDao.java", null); - assertThat(filter.accept(indexedFile, InputFile.Type.MAIN)).isTrue(); + assertThat(filter.accept(indexedFile.path(), Paths.get(indexedFile.relativePath()), InputFile.Type.MAIN)).isTrue(); indexedFile = new DefaultIndexedFile("foo", moduleBaseDir, "src/main/java/com/mycompany/Foo.java", null); - assertThat(filter.accept(indexedFile, InputFile.Type.MAIN)).isFalse(); + assertThat(filter.accept(indexedFile.path(), Paths.get(indexedFile.relativePath()), InputFile.Type.MAIN)).isFalse(); } @Test @@ -78,10 +78,10 @@ public class ExclusionFiltersTest { filter.prepare(); IndexedFile indexedFile = new DefaultIndexedFile("foo", moduleBaseDir, "src/main/java/com/mycompany/Foo.java", null); - assertThat(filter.accept(indexedFile, InputFile.Type.MAIN)).isFalse(); + assertThat(filter.accept(indexedFile.path(), Paths.get(indexedFile.relativePath()), InputFile.Type.MAIN)).isFalse(); indexedFile = new DefaultIndexedFile("foo", moduleBaseDir, "src/main/java/com/mycompany/FooDto.java", null); - assertThat(filter.accept(indexedFile, InputFile.Type.MAIN)).isTrue(); + assertThat(filter.accept(indexedFile.path(), Paths.get(indexedFile.relativePath()), InputFile.Type.MAIN)).isTrue(); } @Test @@ -92,14 +92,14 @@ public class ExclusionFiltersTest { filter.prepare(); IndexedFile indexedFile = new DefaultIndexedFile("foo", moduleBaseDir, "src/main/java/com/mycompany/FooDao.java", null); - assertThat(filter.accept(indexedFile, InputFile.Type.MAIN)).isFalse(); + assertThat(filter.accept(indexedFile.path(), Paths.get(indexedFile.relativePath()), InputFile.Type.MAIN)).isFalse(); indexedFile = new DefaultIndexedFile("foo", moduleBaseDir, "src/main/java/com/mycompany/Foo.java", null); - assertThat(filter.accept(indexedFile, InputFile.Type.MAIN)).isTrue(); + assertThat(filter.accept(indexedFile.path(), Paths.get(indexedFile.relativePath()), InputFile.Type.MAIN)).isTrue(); // source exclusions do not apply to tests indexedFile = new DefaultIndexedFile("foo", moduleBaseDir, "src/test/java/com/mycompany/FooDao.java", null); - assertThat(filter.accept(indexedFile, InputFile.Type.TEST)).isTrue(); + assertThat(filter.accept(indexedFile.path(), Paths.get(indexedFile.relativePath()), InputFile.Type.TEST)).isTrue(); } @Test @@ -111,10 +111,10 @@ public class ExclusionFiltersTest { filter.prepare(); IndexedFile indexedFile = new DefaultIndexedFile("foo", moduleBaseDir, "src/main/java/org/bar/Foo.java", null); - assertThat(filter.accept(indexedFile, InputFile.Type.MAIN)).isTrue(); + assertThat(filter.accept(indexedFile.path(), Paths.get(indexedFile.relativePath()), InputFile.Type.MAIN)).isTrue(); indexedFile = new DefaultIndexedFile("foo", moduleBaseDir, "src/main/java/org/bar/Bar.java", null); - assertThat(filter.accept(indexedFile, InputFile.Type.MAIN)).isFalse(); + assertThat(filter.accept(indexedFile.path(), Paths.get(indexedFile.relativePath()), InputFile.Type.MAIN)).isFalse(); } @Test diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileBuilderTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileBuilderTest.java index a387635db29..a4c6a32b6dd 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileBuilderTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileBuilderTest.java @@ -53,19 +53,18 @@ public class InputFileBuilderTest { .setWorkDir(workDir.toFile()) .setKey("module1"), 0); - LanguageDetection langDetection = mock(LanguageDetection.class); MetadataGenerator metadataGenerator = mock(MetadataGenerator.class); BatchIdGenerator idGenerator = new BatchIdGenerator(); MapSettings settings = new MapSettings(); ModuleFileSystemInitializer moduleFileSystemInitializer = mock(ModuleFileSystemInitializer.class); when(moduleFileSystemInitializer.defaultEncoding()).thenReturn(StandardCharsets.UTF_8); - builder = new InputFileBuilder(module, langDetection, metadataGenerator, idGenerator, settings.asConfig(), moduleFileSystemInitializer); + builder = new InputFileBuilder(module, metadataGenerator, idGenerator, settings.asConfig(), moduleFileSystemInitializer); } @Test public void testBuild() { Path filePath = baseDir.resolve("src/File1.xoo"); - DefaultInputFile inputFile = builder.create(filePath, Type.MAIN); + DefaultInputFile inputFile = builder.create(Type.MAIN, "src/File1.xoo", null); assertThat(inputFile.moduleKey()).isEqualTo("module1"); assertThat(inputFile.absolutePath()).isEqualTo(filePath.toString().replaceAll("\\\\", "/")); diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java index 1d8f553d1f5..1e516523acf 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java @@ -19,13 +19,9 @@ */ package org.sonar.scanner.scan.filesystem; -import static junit.framework.Assert.fail; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.spy; - import java.io.File; import java.io.IOException; - +import java.nio.file.Paths; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -39,6 +35,10 @@ import org.sonar.api.utils.MessageException; import org.sonar.scanner.repository.language.DefaultLanguagesRepository; import org.sonar.scanner.repository.language.LanguagesRepository; +import static junit.framework.Assert.fail; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.spy; + public class LanguageDetectionTest { @Rule @@ -183,7 +183,7 @@ public class LanguageDetectionTest { } private String detectLanguage(LanguageDetection detection, String path) { - return detection.language(new File(temp.getRoot(), path).getAbsolutePath(), path); + return detection.language(new File(temp.getRoot(), path).toPath(), Paths.get(path)); } static class MockLanguage implements Language { |