SONAR-19423 Files without suffixes can be declared as belonging to a Language

diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/DefaultLanguagesRepository.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/DefaultLanguagesRepository.java
index ca5a9586765cebe8274db8b0214bad270d3f62ef..353c7b2d492ba92a885508c2fd9ff5f9d6653835 100644 (file)
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/DefaultLanguagesRepository.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/DefaultLanguagesRepository.java
@@ -19,8 +19,9 @@
  */
 package org.sonar.scanner.repository.language;
 
-import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
+import java.util.stream.Collectors;
 import javax.annotation.CheckForNull;
 import javax.annotation.concurrent.Immutable;
 import org.sonar.api.Startable;
@@ -33,7 +34,7 @@ import org.sonar.api.resources.Languages;
 @Immutable
 public class DefaultLanguagesRepository implements LanguagesRepository, Startable {
 
-  private Languages languages;
+  private final Languages languages;
 
   public DefaultLanguagesRepository(Languages languages) {
     this.languages = languages;
@@ -53,7 +54,7 @@ public class DefaultLanguagesRepository implements LanguagesRepository, Startabl
   @CheckForNull
   public Language get(String languageKey) {
     org.sonar.api.resources.Language language = languages.get(languageKey);
-    return language != null ? new Language(language.getKey(), language.getName(), language.publishAllFiles(), language.getFileSuffixes()) : null;
+    return language != null ? new Language(language) : null;
   }
 
   /**
@@ -61,12 +62,9 @@ public class DefaultLanguagesRepository implements LanguagesRepository, Startabl
    */
   @Override
   public Collection<Language> all() {
-    org.sonar.api.resources.Language[] all = languages.all();
-    Collection<Language> result = new ArrayList<>(all.length);
-    for (org.sonar.api.resources.Language language : all) {
-      result.add(new Language(language.getKey(), language.getName(), language.publishAllFiles(), language.getFileSuffixes()));
-    }
-    return result;
+    return Arrays.stream(languages.all())
+      .map(Language::new)
+      .collect(Collectors.toList());
   }
 
   @Override

diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/Language.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/Language.java
index d1f9edfc3ffb045f3e102511d3b844efd1d8d4b5..a508e56ef0a8ff906cf312dbace597aeebb07b1f 100644 (file)
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/Language.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/language/Language.java
@@ -31,12 +31,15 @@ public final class Language {
   private final String name;
   private final boolean publishAllFiles;
   private final String[] fileSuffixes;
+  private final String[] filenamePatterns;
 
-  public Language(String key, String name, boolean publishAllFiles, String... fileSuffixes) {
-    this.key = key;
-    this.name = name;
-    this.publishAllFiles = publishAllFiles;
-    this.fileSuffixes = fileSuffixes;
+
+  public Language(org.sonar.api.resources.Language language) {
+    this.key = language.getKey();
+    this.name = language.getName();
+    this.publishAllFiles = language.publishAllFiles();
+    this.fileSuffixes = language.getFileSuffixes();
+    this.filenamePatterns = language.filenamePatterns();
   }
 
   /**
@@ -60,6 +63,10 @@ public final class Language {
     return Arrays.asList(fileSuffixes);
   }
 
+  public Collection<String> filenamePatterns() {
+    return Arrays.asList(filenamePatterns);
+  }
+
   public boolean isPublishAllFiles() {
     return publishAllFiles;
   }

diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java
index 3822ab46b6a85074882fec790e5bed5153c92e0c..5ede0250c087b2721c7a1b41ca03c9a088ea6264 100644 (file)
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/LanguageDetection.java
@@ -26,6 +26,7 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import javax.annotation.CheckForNull;
 import javax.annotation.concurrent.ThreadSafe;
 import org.apache.commons.lang.StringUtils;
@@ -61,16 +62,8 @@ public class LanguageDetection {
       if (pathPatterns.length > 0) {
         patternsByLanguageBuilder.put(language, pathPatterns);
       } else {
-        // If no custom language pattern is defined then fallback to suffixes declared by language
-        String[] patterns = language.fileSuffixes().toArray(new String[0]);
-        for (int i = 0; i < patterns.length; i++) {
-          String suffix = patterns[i];
-          String extension = sanitizeExtension(suffix);
-          patterns[i] = "**/*." + extension;
-        }
-        PathPattern[] defaultLanguagePatterns = PathPattern.create(patterns);
-        patternsByLanguageBuilder.put(language, defaultLanguagePatterns);
-        LOG.debug("Declared extensions of language {} were converted to {}", language, getDetails(language, defaultLanguagePatterns));
+        PathPattern[] languagePatterns = getLanguagePatterns(language);
+        patternsByLanguageBuilder.put(language, languagePatterns);
       }
     }
 
@@ -78,6 +71,22 @@ public class LanguageDetection {
     patternsByLanguage = unmodifiableMap(patternsByLanguageBuilder);
   }
 
+  private static PathPattern[] getLanguagePatterns(Language language) {
+    Stream<PathPattern> fileSuffixes = language.fileSuffixes().stream()
+      .map(suffix -> "**/*." + sanitizeExtension(suffix))
+      .map(PathPattern::create);
+    Stream<PathPattern> filenamePatterns = language.filenamePatterns()
+      .stream()
+      .map(filenamePattern -> "**/" + filenamePattern)
+      .map(PathPattern::create);
+
+    PathPattern[] defaultLanguagePatterns = Stream.concat(fileSuffixes, filenamePatterns)
+      .distinct()
+      .toArray(PathPattern[]::new);
+    LOG.debug("Declared patterns of language {} were converted to {}", language, getDetails(language, defaultLanguagePatterns));
+    return defaultLanguagePatterns;
+  }
+
   @CheckForNull
   Language language(Path absolutePath, Path relativePath) {
     Language detectedLanguage = null;
@@ -98,14 +107,7 @@ public class LanguageDetection {
 
   private boolean isCandidateForLanguage(Path absolutePath, Path relativePath, Language language) {
     PathPattern[] patterns = patternsByLanguage.get(language);
-    if (patterns != null) {
-      for (PathPattern pathPattern : patterns) {
-        if (pathPattern.match(absolutePath, relativePath, false)) {
-          return true;
-        }
-      }
-    }
-    return false;
+    return patterns != null && Arrays.stream(patterns).anyMatch(pattern -> pattern.match(absolutePath, relativePath, false));
   }
 
   private static String getFileLangPatternPropKey(String languageKey) {

diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/repository/language/LanguageTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/repository/language/LanguageTest.java
index f3f078b3d3c8f1c2e22b5848e3a6c8c9eeea1882..1d49dbbe1204174defea4ce948accac07b36d3dd 100644 (file)
--- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/repository/language/LanguageTest.java
+++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/repository/language/LanguageTest.java
@@ -22,18 +22,20 @@ package org.sonar.scanner.repository.language;
 import org.junit.Test;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class LanguageTest {
   @Test
   public void hashCode_and_equals_depends_on_key() {
-    Language lang1 = new Language("key1", "name1", true, "f1");
-    Language lang2 = new Language("key1", "name2", false, "f2");
-    Language lang3 = new Language("key2", "name1", true, "f1");
+    Language lang1 = new Language(mockApiLanguage("key1", "name1", true, new String[] {"f1"}, new String[0]));
+    Language lang2 = new Language(mockApiLanguage("key1", "name2", false, new String[] {"f2"}, new String[0]));
+    Language lang3 = new Language(mockApiLanguage("key2", "name1", true, new String[] {"f1"}, new String[0]));
     assertThat(lang1)
       .hasSameHashCodeAs(lang2)
       .doesNotHaveSameHashCodeAs(lang3);
     assertThat(lang2).doesNotHaveSameHashCodeAs(lang3);
-    
+
     assertThat(lang1)
       .isEqualTo(lang2)
       .isNotEqualTo(lang3);
@@ -42,10 +44,21 @@ public class LanguageTest {
 
   @Test
   public void getters_match_constructor() {
-    Language lang1 = new Language("key1", "name1", true, "f1");
+    Language lang1 = new Language(mockApiLanguage("key1", "name1", true, new String[] {"f1"}, new String[] {"p1"}));
     assertThat(lang1.key()).isEqualTo("key1");
     assertThat(lang1.name()).isEqualTo("name1");
     assertThat(lang1.isPublishAllFiles()).isTrue();
     assertThat(lang1.fileSuffixes()).containsOnly("f1");
+    assertThat(lang1.filenamePatterns()).containsOnly("p1");
+  }
+
+  private org.sonar.api.resources.Language mockApiLanguage(String key, String name, boolean publishAllFiles, String[] fileSuffixes, String[] filenamePatterns) {
+    org.sonar.api.resources.Language mock = mock(org.sonar.api.resources.Language.class);
+    when(mock.getKey()).thenReturn(key);
+    when(mock.getName()).thenReturn(name);
+    when(mock.publishAllFiles()).thenReturn(publishAllFiles);
+    when(mock.getFileSuffixes()).thenReturn(fileSuffixes);
+    when(mock.filenamePatterns()).thenReturn(filenamePatterns);
+    return mock;
   }
 }

diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java
index 03f1547bab2d123f1817269178c1920a91f13042..c1136a573fd0b735ba8a3b1da838c7b7036c2ef7 100644 (file)
--- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java
+++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java
@@ -19,12 +19,16 @@
  */
 package org.sonar.scanner.scan.filesystem;
 
+import com.tngtech.java.junit.dataprovider.DataProvider;
+import com.tngtech.java.junit.dataprovider.DataProviderRunner;
+import com.tngtech.java.junit.dataprovider.UseDataProvider;
 import java.io.File;
 import java.nio.file.Paths;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
 import org.sonar.api.config.internal.MapSettings;
 import org.sonar.api.resources.Language;
 import org.sonar.api.resources.Languages;
@@ -32,10 +36,11 @@ import org.sonar.api.utils.MessageException;
 import org.sonar.scanner.repository.language.DefaultLanguagesRepository;
 import org.sonar.scanner.repository.language.LanguagesRepository;
 
-import static junit.framework.Assert.fail;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.mockito.Mockito.spy;
 
+@RunWith(DataProviderRunner.class)
 public class LanguageDetectionTest {
 
   @Rule
@@ -49,15 +54,23 @@ public class LanguageDetectionTest {
   }
 
   @Test
-  public void test_sanitizeExtension() {
-    assertThat(LanguageDetection.sanitizeExtension(".cbl")).isEqualTo("cbl");
-    assertThat(LanguageDetection.sanitizeExtension(".CBL")).isEqualTo("cbl");
-    assertThat(LanguageDetection.sanitizeExtension("CBL")).isEqualTo("cbl");
-    assertThat(LanguageDetection.sanitizeExtension("cbl")).isEqualTo("cbl");
+  @UseDataProvider("extensionsForSanitization")
+  public void sanitizeExtension_shouldRemoveObsoleteCharacters(String extension) {
+    assertThat(LanguageDetection.sanitizeExtension(extension)).isEqualTo("cbl");
+  }
+
+  @DataProvider
+  public static Object[][] extensionsForSanitization() {
+    return new Object[][] {
+      {".cbl"},
+      {".CBL"},
+      {"CBL"},
+      {"cbl"},
+    };
   }
 
   @Test
-  public void search_by_file_extension() {
+  public void detectLanguageKey_shouldDetectByFileExtension() {
     LanguagesRepository languages = new DefaultLanguagesRepository(new Languages(new MockLanguage("java", "java", "jav"), new MockLanguage("cobol", "cbl", "cob")));
     LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
 
@@ -75,13 +88,41 @@ public class LanguageDetectionTest {
   }
 
   @Test
-  public void should_not_fail_if_no_language() {
+  @UseDataProvider("filenamePatterns")
+  public void detectLanguageKey_shouldDetectByFileNamePattern(String fileName, String expectedLanguageKey) {
+    LanguagesRepository languages = new DefaultLanguagesRepository(new Languages(
+      new MockLanguage("docker", new String[0], new String[] {"*.dockerfile", "*.Dockerfile", "Dockerfile", "Dockerfile.*"}),
+      new MockLanguage("terraform", new String[] {"tf"}, new String[] {".tf"}),
+      new MockLanguage("java", new String[0], new String[] {"**/*Test.java"})));
+    LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
+    assertThat(detectLanguageKey(detection, fileName)).isEqualTo(expectedLanguageKey);
+  }
+
+  @DataProvider
+  public static Object[][] filenamePatterns() {
+    return new Object[][] {
+      {"Dockerfile", "docker"},
+      {"src/Dockerfile", "docker"},
+      {"my.Dockerfile", "docker"},
+      {"my.dockerfile", "docker"},
+      {"Dockerfile.old", "docker"},
+      {"Dockerfile.OLD", "docker"},
+      {"DOCKERFILE", null},
+      {"infra.tf", "terraform"},
+      {"FooTest.java", "java"},
+      {"FooTest.JAVA", "java"},
+      {"FooTEST.java", null}
+    };
+  }
+
+  @Test
+  public void detectLanguageKey_shouldNotFailIfNoLanguage() {
     LanguageDetection detection = spy(new LanguageDetection(settings.asConfig(), new DefaultLanguagesRepository(new Languages())));
     assertThat(detectLanguageKey(detection, "Foo.java")).isNull();
   }
 
   @Test
-  public void plugin_can_declare_a_file_extension_twice_for_case_sensitivity() {
+  public void detectLanguageKey_shouldAllowPluginsToDeclareFileExtensionTwiceForCaseSensitivity() {
     LanguagesRepository languages = new DefaultLanguagesRepository(new Languages(new MockLanguage("abap", "abap", "ABAP")));
 
     LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
@@ -89,22 +130,18 @@ public class LanguageDetectionTest {
   }
 
   @Test
-  public void fail_if_conflicting_language_suffix() {
+  public void detectLanguageKey_shouldFailIfConflictingLanguageSuffix() {
     LanguagesRepository languages = new DefaultLanguagesRepository(new Languages(new MockLanguage("xml", "xhtml"), new MockLanguage("web", "xhtml")));
     LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
-    try {
-      detectLanguageKey(detection, "abc.xhtml");
-      fail();
-    } catch (MessageException e) {
-      assertThat(e.getMessage())
-        .contains("Language of file 'abc.xhtml' can not be decided as the file matches patterns of both ")
-        .contains("sonar.lang.patterns.web : **/*.xhtml")
-        .contains("sonar.lang.patterns.xml : **/*.xhtml");
-    }
+    assertThatThrownBy(() -> detectLanguageKey(detection, "abc.xhtml"))
+      .isInstanceOf(MessageException.class)
+      .hasMessageContaining("Language of file 'abc.xhtml' can not be decided as the file matches patterns of both ")
+      .hasMessageContaining("sonar.lang.patterns.web : **/*.xhtml")
+      .hasMessageContaining("sonar.lang.patterns.xml : **/*.xhtml");
   }
 
   @Test
-  public void solve_conflict_using_filepattern() {
+  public void detectLanguageKey_shouldSolveConflictUsingFilePattern() {
     LanguagesRepository languages = new DefaultLanguagesRepository(new Languages(new MockLanguage("xml", "xhtml"), new MockLanguage("web", "xhtml")));
 
     settings.setProperty("sonar.lang.patterns.xml", "xml/**");
@@ -115,7 +152,7 @@ public class LanguageDetectionTest {
   }
 
   @Test
-  public void fail_if_conflicting_filepattern() {
+  public void detectLanguageKey_shouldFailIfConflictingFilePattern() {
     LanguagesRepository languages = new DefaultLanguagesRepository(new Languages(new MockLanguage("abap", "abap"), new MockLanguage("cobol", "cobol")));
     settings.setProperty("sonar.lang.patterns.abap", "*.abap,*.txt");
     settings.setProperty("sonar.lang.patterns.cobol", "*.cobol,*.txt");
@@ -124,15 +161,11 @@ public class LanguageDetectionTest {
 
     assertThat(detectLanguageKey(detection, "abc.abap")).isEqualTo("abap");
     assertThat(detectLanguageKey(detection, "abc.cobol")).isEqualTo("cobol");
-    try {
-      detectLanguageKey(detection, "abc.txt");
-      fail();
-    } catch (MessageException e) {
-      assertThat(e.getMessage())
-        .contains("Language of file 'abc.txt' can not be decided as the file matches patterns of both ")
-        .contains("sonar.lang.patterns.abap : *.abap,*.txt")
-        .contains("sonar.lang.patterns.cobol : *.cobol,*.txt");
-    }
+
+    assertThatThrownBy(() -> detectLanguageKey(detection, "abc.txt"))
+      .hasMessageContaining("Language of file 'abc.txt' can not be decided as the file matches patterns of both ")
+      .hasMessageContaining("sonar.lang.patterns.abap : *.abap,*.txt")
+      .hasMessageContaining("sonar.lang.patterns.cobol : *.cobol,*.txt");
   }
 
   private String detectLanguageKey(LanguageDetection detection, String path) {
@@ -143,10 +176,18 @@ public class LanguageDetectionTest {
   static class MockLanguage implements Language {
     private final String key;
     private final String[] extensions;
+    private final String[] filenamePatterns;
 
     MockLanguage(String key, String... extensions) {
       this.key = key;
       this.extensions = extensions;
+      this.filenamePatterns = new String[0];
+    }
+
+    MockLanguage(String key, String[] extensions, String[] filenamePatterns) {
+      this.key = key;
+      this.extensions = extensions;
+      this.filenamePatterns = filenamePatterns;
     }
 
     @Override
@@ -164,6 +205,11 @@ public class LanguageDetectionTest {
       return extensions;
     }
 
+    @Override
+    public String[] filenamePatterns() {
+      return filenamePatterns;
+    }
+
     @Override
     public boolean publishAllFiles() {
       return true;