From: Jacek Poreda Date: Wed, 31 May 2023 14:01:25 +0000 (+0200) Subject: SONAR-19423 Files without suffixes can be declared as belonging to a Language X-Git-Tag: 10.1.0.73491~178 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=3bcee0f3a74e7a5b144ed438f50b4239ee7339f7;p=sonarqube.git SONAR-19423 Files without suffixes can be declared as belonging to a Language --- diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/DefaultLanguagesRepository.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/DefaultLanguagesRepository.java index ca5a9586765..353c7b2d492 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/DefaultLanguagesRepository.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/DefaultLanguagesRepository.java @@ -19,8 +19,9 @@ */ package org.sonar.scanner.repository.language; -import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; +import java.util.stream.Collectors; import javax.annotation.CheckForNull; import javax.annotation.concurrent.Immutable; import org.sonar.api.Startable; @@ -33,7 +34,7 @@ import org.sonar.api.resources.Languages; @Immutable public class DefaultLanguagesRepository implements LanguagesRepository, Startable { - private Languages languages; + private final Languages languages; public DefaultLanguagesRepository(Languages languages) { this.languages = languages; @@ -53,7 +54,7 @@ public class DefaultLanguagesRepository implements LanguagesRepository, Startabl @CheckForNull public Language get(String languageKey) { org.sonar.api.resources.Language language = languages.get(languageKey); - return language != null ? new Language(language.getKey(), language.getName(), language.publishAllFiles(), language.getFileSuffixes()) : null; + return language != null ? new Language(language) : null; } /** @@ -61,12 +62,9 @@ public class DefaultLanguagesRepository implements LanguagesRepository, Startabl */ @Override public Collection all() { - org.sonar.api.resources.Language[] all = languages.all(); - Collection result = new ArrayList<>(all.length); - for (org.sonar.api.resources.Language language : all) { - result.add(new Language(language.getKey(), language.getName(), language.publishAllFiles(), language.getFileSuffixes())); - } - return result; + return Arrays.stream(languages.all()) + .map(Language::new) + .collect(Collectors.toList()); } @Override diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/Language.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/Language.java index d1f9edfc3ff..a508e56ef0a 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/Language.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/Language.java @@ -31,12 +31,15 @@ public final class Language { private final String name; private final boolean publishAllFiles; private final String[] fileSuffixes; + private final String[] filenamePatterns; - public Language(String key, String name, boolean publishAllFiles, String... fileSuffixes) { - this.key = key; - this.name = name; - this.publishAllFiles = publishAllFiles; - this.fileSuffixes = fileSuffixes; + + public Language(org.sonar.api.resources.Language language) { + this.key = language.getKey(); + this.name = language.getName(); + this.publishAllFiles = language.publishAllFiles(); + this.fileSuffixes = language.getFileSuffixes(); + this.filenamePatterns = language.filenamePatterns(); } /** @@ -60,6 +63,10 @@ public final class Language { return Arrays.asList(fileSuffixes); } + public Collection filenamePatterns() { + return Arrays.asList(filenamePatterns); + } + public boolean isPublishAllFiles() { return publishAllFiles; } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java index 3822ab46b6a..5ede0250c08 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java @@ -26,6 +26,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.CheckForNull; import javax.annotation.concurrent.ThreadSafe; import org.apache.commons.lang.StringUtils; @@ -61,16 +62,8 @@ public class LanguageDetection { if (pathPatterns.length > 0) { patternsByLanguageBuilder.put(language, pathPatterns); } else { - // If no custom language pattern is defined then fallback to suffixes declared by language - String[] patterns = language.fileSuffixes().toArray(new String[0]); - for (int i = 0; i < patterns.length; i++) { - String suffix = patterns[i]; - String extension = sanitizeExtension(suffix); - patterns[i] = "**/*." + extension; - } - PathPattern[] defaultLanguagePatterns = PathPattern.create(patterns); - patternsByLanguageBuilder.put(language, defaultLanguagePatterns); - LOG.debug("Declared extensions of language {} were converted to {}", language, getDetails(language, defaultLanguagePatterns)); + PathPattern[] languagePatterns = getLanguagePatterns(language); + patternsByLanguageBuilder.put(language, languagePatterns); } } @@ -78,6 +71,22 @@ public class LanguageDetection { patternsByLanguage = unmodifiableMap(patternsByLanguageBuilder); } + private static PathPattern[] getLanguagePatterns(Language language) { + Stream fileSuffixes = language.fileSuffixes().stream() + .map(suffix -> "**/*." + sanitizeExtension(suffix)) + .map(PathPattern::create); + Stream filenamePatterns = language.filenamePatterns() + .stream() + .map(filenamePattern -> "**/" + filenamePattern) + .map(PathPattern::create); + + PathPattern[] defaultLanguagePatterns = Stream.concat(fileSuffixes, filenamePatterns) + .distinct() + .toArray(PathPattern[]::new); + LOG.debug("Declared patterns of language {} were converted to {}", language, getDetails(language, defaultLanguagePatterns)); + return defaultLanguagePatterns; + } + @CheckForNull Language language(Path absolutePath, Path relativePath) { Language detectedLanguage = null; @@ -98,14 +107,7 @@ public class LanguageDetection { private boolean isCandidateForLanguage(Path absolutePath, Path relativePath, Language language) { PathPattern[] patterns = patternsByLanguage.get(language); - if (patterns != null) { - for (PathPattern pathPattern : patterns) { - if (pathPattern.match(absolutePath, relativePath, false)) { - return true; - } - } - } - return false; + return patterns != null && Arrays.stream(patterns).anyMatch(pattern -> pattern.match(absolutePath, relativePath, false)); } private static String getFileLangPatternPropKey(String languageKey) { diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/repository/language/LanguageTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/repository/language/LanguageTest.java index f3f078b3d3c..1d49dbbe120 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/repository/language/LanguageTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/repository/language/LanguageTest.java @@ -22,18 +22,20 @@ package org.sonar.scanner.repository.language; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class LanguageTest { @Test public void hashCode_and_equals_depends_on_key() { - Language lang1 = new Language("key1", "name1", true, "f1"); - Language lang2 = new Language("key1", "name2", false, "f2"); - Language lang3 = new Language("key2", "name1", true, "f1"); + Language lang1 = new Language(mockApiLanguage("key1", "name1", true, new String[] {"f1"}, new String[0])); + Language lang2 = new Language(mockApiLanguage("key1", "name2", false, new String[] {"f2"}, new String[0])); + Language lang3 = new Language(mockApiLanguage("key2", "name1", true, new String[] {"f1"}, new String[0])); assertThat(lang1) .hasSameHashCodeAs(lang2) .doesNotHaveSameHashCodeAs(lang3); assertThat(lang2).doesNotHaveSameHashCodeAs(lang3); - + assertThat(lang1) .isEqualTo(lang2) .isNotEqualTo(lang3); @@ -42,10 +44,21 @@ public class LanguageTest { @Test public void getters_match_constructor() { - Language lang1 = new Language("key1", "name1", true, "f1"); + Language lang1 = new Language(mockApiLanguage("key1", "name1", true, new String[] {"f1"}, new String[] {"p1"})); assertThat(lang1.key()).isEqualTo("key1"); assertThat(lang1.name()).isEqualTo("name1"); assertThat(lang1.isPublishAllFiles()).isTrue(); assertThat(lang1.fileSuffixes()).containsOnly("f1"); + assertThat(lang1.filenamePatterns()).containsOnly("p1"); + } + + private org.sonar.api.resources.Language mockApiLanguage(String key, String name, boolean publishAllFiles, String[] fileSuffixes, String[] filenamePatterns) { + org.sonar.api.resources.Language mock = mock(org.sonar.api.resources.Language.class); + when(mock.getKey()).thenReturn(key); + when(mock.getName()).thenReturn(name); + when(mock.publishAllFiles()).thenReturn(publishAllFiles); + when(mock.getFileSuffixes()).thenReturn(fileSuffixes); + when(mock.filenamePatterns()).thenReturn(filenamePatterns); + return mock; } } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java index 03f1547bab2..c1136a573fd 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java @@ -19,12 +19,16 @@ */ package org.sonar.scanner.scan.filesystem; +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; import java.io.File; import java.nio.file.Paths; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; import org.sonar.api.config.internal.MapSettings; import org.sonar.api.resources.Language; import org.sonar.api.resources.Languages; @@ -32,10 +36,11 @@ import org.sonar.api.utils.MessageException; import org.sonar.scanner.repository.language.DefaultLanguagesRepository; import org.sonar.scanner.repository.language.LanguagesRepository; -import static junit.framework.Assert.fail; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.spy; +@RunWith(DataProviderRunner.class) public class LanguageDetectionTest { @Rule @@ -49,15 +54,23 @@ public class LanguageDetectionTest { } @Test - public void test_sanitizeExtension() { - assertThat(LanguageDetection.sanitizeExtension(".cbl")).isEqualTo("cbl"); - assertThat(LanguageDetection.sanitizeExtension(".CBL")).isEqualTo("cbl"); - assertThat(LanguageDetection.sanitizeExtension("CBL")).isEqualTo("cbl"); - assertThat(LanguageDetection.sanitizeExtension("cbl")).isEqualTo("cbl"); + @UseDataProvider("extensionsForSanitization") + public void sanitizeExtension_shouldRemoveObsoleteCharacters(String extension) { + assertThat(LanguageDetection.sanitizeExtension(extension)).isEqualTo("cbl"); + } + + @DataProvider + public static Object[][] extensionsForSanitization() { + return new Object[][] { + {".cbl"}, + {".CBL"}, + {"CBL"}, + {"cbl"}, + }; } @Test - public void search_by_file_extension() { + public void detectLanguageKey_shouldDetectByFileExtension() { LanguagesRepository languages = new DefaultLanguagesRepository(new Languages(new MockLanguage("java", "java", "jav"), new MockLanguage("cobol", "cbl", "cob"))); LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages); @@ -75,13 +88,41 @@ public class LanguageDetectionTest { } @Test - public void should_not_fail_if_no_language() { + @UseDataProvider("filenamePatterns") + public void detectLanguageKey_shouldDetectByFileNamePattern(String fileName, String expectedLanguageKey) { + LanguagesRepository languages = new DefaultLanguagesRepository(new Languages( + new MockLanguage("docker", new String[0], new String[] {"*.dockerfile", "*.Dockerfile", "Dockerfile", "Dockerfile.*"}), + new MockLanguage("terraform", new String[] {"tf"}, new String[] {".tf"}), + new MockLanguage("java", new String[0], new String[] {"**/*Test.java"}))); + LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages); + assertThat(detectLanguageKey(detection, fileName)).isEqualTo(expectedLanguageKey); + } + + @DataProvider + public static Object[][] filenamePatterns() { + return new Object[][] { + {"Dockerfile", "docker"}, + {"src/Dockerfile", "docker"}, + {"my.Dockerfile", "docker"}, + {"my.dockerfile", "docker"}, + {"Dockerfile.old", "docker"}, + {"Dockerfile.OLD", "docker"}, + {"DOCKERFILE", null}, + {"infra.tf", "terraform"}, + {"FooTest.java", "java"}, + {"FooTest.JAVA", "java"}, + {"FooTEST.java", null} + }; + } + + @Test + public void detectLanguageKey_shouldNotFailIfNoLanguage() { LanguageDetection detection = spy(new LanguageDetection(settings.asConfig(), new DefaultLanguagesRepository(new Languages()))); assertThat(detectLanguageKey(detection, "Foo.java")).isNull(); } @Test - public void plugin_can_declare_a_file_extension_twice_for_case_sensitivity() { + public void detectLanguageKey_shouldAllowPluginsToDeclareFileExtensionTwiceForCaseSensitivity() { LanguagesRepository languages = new DefaultLanguagesRepository(new Languages(new MockLanguage("abap", "abap", "ABAP"))); LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages); @@ -89,22 +130,18 @@ public class LanguageDetectionTest { } @Test - public void fail_if_conflicting_language_suffix() { + public void detectLanguageKey_shouldFailIfConflictingLanguageSuffix() { LanguagesRepository languages = new DefaultLanguagesRepository(new Languages(new MockLanguage("xml", "xhtml"), new MockLanguage("web", "xhtml"))); LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages); - try { - detectLanguageKey(detection, "abc.xhtml"); - fail(); - } catch (MessageException e) { - assertThat(e.getMessage()) - .contains("Language of file 'abc.xhtml' can not be decided as the file matches patterns of both ") - .contains("sonar.lang.patterns.web : **/*.xhtml") - .contains("sonar.lang.patterns.xml : **/*.xhtml"); - } + assertThatThrownBy(() -> detectLanguageKey(detection, "abc.xhtml")) + .isInstanceOf(MessageException.class) + .hasMessageContaining("Language of file 'abc.xhtml' can not be decided as the file matches patterns of both ") + .hasMessageContaining("sonar.lang.patterns.web : **/*.xhtml") + .hasMessageContaining("sonar.lang.patterns.xml : **/*.xhtml"); } @Test - public void solve_conflict_using_filepattern() { + public void detectLanguageKey_shouldSolveConflictUsingFilePattern() { LanguagesRepository languages = new DefaultLanguagesRepository(new Languages(new MockLanguage("xml", "xhtml"), new MockLanguage("web", "xhtml"))); settings.setProperty("sonar.lang.patterns.xml", "xml/**"); @@ -115,7 +152,7 @@ public class LanguageDetectionTest { } @Test - public void fail_if_conflicting_filepattern() { + public void detectLanguageKey_shouldFailIfConflictingFilePattern() { LanguagesRepository languages = new DefaultLanguagesRepository(new Languages(new MockLanguage("abap", "abap"), new MockLanguage("cobol", "cobol"))); settings.setProperty("sonar.lang.patterns.abap", "*.abap,*.txt"); settings.setProperty("sonar.lang.patterns.cobol", "*.cobol,*.txt"); @@ -124,15 +161,11 @@ public class LanguageDetectionTest { assertThat(detectLanguageKey(detection, "abc.abap")).isEqualTo("abap"); assertThat(detectLanguageKey(detection, "abc.cobol")).isEqualTo("cobol"); - try { - detectLanguageKey(detection, "abc.txt"); - fail(); - } catch (MessageException e) { - assertThat(e.getMessage()) - .contains("Language of file 'abc.txt' can not be decided as the file matches patterns of both ") - .contains("sonar.lang.patterns.abap : *.abap,*.txt") - .contains("sonar.lang.patterns.cobol : *.cobol,*.txt"); - } + + assertThatThrownBy(() -> detectLanguageKey(detection, "abc.txt")) + .hasMessageContaining("Language of file 'abc.txt' can not be decided as the file matches patterns of both ") + .hasMessageContaining("sonar.lang.patterns.abap : *.abap,*.txt") + .hasMessageContaining("sonar.lang.patterns.cobol : *.cobol,*.txt"); } private String detectLanguageKey(LanguageDetection detection, String path) { @@ -143,10 +176,18 @@ public class LanguageDetectionTest { static class MockLanguage implements Language { private final String key; private final String[] extensions; + private final String[] filenamePatterns; MockLanguage(String key, String... extensions) { this.key = key; this.extensions = extensions; + this.filenamePatterns = new String[0]; + } + + MockLanguage(String key, String[] extensions, String[] filenamePatterns) { + this.key = key; + this.extensions = extensions; + this.filenamePatterns = filenamePatterns; } @Override @@ -164,6 +205,11 @@ public class LanguageDetectionTest { return extensions; } + @Override + public String[] filenamePatterns() { + return filenamePatterns; + } + @Override public boolean publishAllFiles() { return true;