SONAR-926 Multi-language support:

* RulesProfile wrapper
* ModuleFileSystem now support multi language
* Sensors executed trying each language until one is found

5 files changed, 231 insertions, 71 deletions

diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ComponentIndexer.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ComponentIndexer.java
new file mode 100644
index 00000000000..ac0bfad02fe
--- /dev/null
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ComponentIndexer.java
@@ -0,0 +1,120 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2013 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.batch.scan.filesystem;
+
+import com.google.common.base.CharMatcher;
+import com.google.common.io.Files;
+import org.apache.commons.lang.StringUtils;
+import org.sonar.api.BatchComponent;
+import org.sonar.api.CoreProperties;
+import org.sonar.api.batch.InstantiationStrategy;
+import org.sonar.api.batch.SonarIndex;
+import org.sonar.api.config.Settings;
+import org.sonar.api.resources.File;
+import org.sonar.api.resources.Java;
+import org.sonar.api.resources.JavaFile;
+import org.sonar.api.resources.Languages;
+import org.sonar.api.resources.Project;
+import org.sonar.api.resources.Resource;
+import org.sonar.api.scan.filesystem.FileQuery;
+import org.sonar.api.scan.filesystem.internal.InputFile;
+import org.sonar.api.utils.SonarException;
+import org.sonar.batch.index.ResourceKeyMigration;
+import org.sonar.batch.scan.language.ModuleLanguages;
+import org.sonar.core.resource.ResourceDao;
+import org.sonar.core.resource.ResourceDto;
+
+/**
+ * Index all files/directories of the module in SQ database and importing source code.
+ * @since 4.2
+ */
+@InstantiationStrategy(InstantiationStrategy.PER_PROJECT)
+public class ComponentIndexer implements BatchComponent {
+
+  private final Languages languages;
+  private final Settings settings;
+  private final SonarIndex sonarIndex;
+  private final ResourceKeyMigration migration;
+  private final Project module;
+  private final ModuleLanguages moduleLanguages;
+  private final ResourceDao resourceDao;
+
+  public ComponentIndexer(Project module, Languages languages, SonarIndex sonarIndex, Settings settings, ResourceKeyMigration migration,
+    ModuleLanguages moduleLanguages, ResourceDao resourceDao) {
+    this.module = module;
+    this.languages = languages;
+    this.sonarIndex = sonarIndex;
+    this.settings = settings;
+    this.migration = migration;
+    this.moduleLanguages = moduleLanguages;
+    this.resourceDao = resourceDao;
+  }
+
+  public void execute(DefaultModuleFileSystem fs) {
+    boolean importSource = settings.getBoolean(CoreProperties.CORE_IMPORT_SOURCES_PROPERTY);
+    Iterable<InputFile> inputFiles = fs.inputFiles(FileQuery.all());
+    migration.migrateIfNeeded(module, inputFiles);
+    for (InputFile inputFile : inputFiles) {
+      String languageKey = inputFile.attribute(InputFile.ATTRIBUTE_LANGUAGE);
+      boolean unitTest = InputFile.TYPE_TEST.equals(inputFile.attribute(InputFile.ATTRIBUTE_TYPE));
+      Resource sonarFile;
+      if (Java.KEY.equals(languageKey)) {
+        sonarFile = JavaFile.create(inputFile.path(), inputFile.attribute(InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH), unitTest);
+      } else {
+        sonarFile = File.create(inputFile.path(), inputFile.attribute(InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH), languages.get(languageKey), unitTest);
+      }
+      if (sonarFile != null) {
+        moduleLanguages.addLanguage(languageKey);
+        sonarIndex.index(sonarFile);
+        if (importSource) {
+          importSources(inputFile, sonarFile);
+        }
+      }
+    }
+
+    updateModuleLanguage();
+  }
+
+  private void importSources(InputFile inputFile, Resource sonarFile) {
+    try {
+      String source = Files.toString(inputFile.file(), inputFile.encoding());
+      // SONAR-3860 Remove BOM character from source
+      source = CharMatcher.anyOf("\uFEFF").removeFrom(source);
+      sonarIndex.setSource(sonarFile, source);
+    } catch (Exception e) {
+      throw new SonarException("Unable to read and import the source file : '" + inputFile.absolutePath() + "' with the charset : '"
+        + inputFile.encoding() + "'.", e);
+    }
+  }
+
+  private void updateModuleLanguage() {
+    if (module.getId() != null) {
+      ResourceDto dto = resourceDao.getResource(module.getId());
+      if (moduleLanguages.getModuleLanguageKeys().size() == 1) {
+        dto.setLanguage(moduleLanguages.getModuleLanguageKeys().iterator().next());
+      } else if (moduleLanguages.getModuleLanguageKeys().size() > 1) {
+        dto.setLanguage(StringUtils.join(moduleLanguages.getModuleLanguageKeys(), ","));
+      } else {
+        dto.setLanguage("none");
+      }
+      resourceDao.insertOrUpdate(dto);
+    }
+  }
+}
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystem.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystem.java
index 4367b8aedd8..80bc58ad5eb 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystem.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystem.java
@@ -23,7 +23,6 @@ import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import org.apache.commons.lang.StringUtils;
-import org.picocontainer.Startable;
 import org.sonar.api.CoreProperties;
 import org.sonar.api.config.Settings;
 import org.sonar.api.resources.Project;
@@ -31,6 +30,7 @@ import org.sonar.api.scan.filesystem.FileQuery;
 import org.sonar.api.scan.filesystem.ModuleFileSystem;
 import org.sonar.api.scan.filesystem.internal.InputFile;
 import org.sonar.api.scan.filesystem.internal.InputFiles;
+import org.sonar.api.utils.SonarException;
 import org.sonar.batch.bootstrap.AnalysisMode;
 
 import javax.annotation.CheckForNull;
@@ -44,7 +44,7 @@ import java.util.List;
  *
  * @since 3.5
  */
-public class DefaultModuleFileSystem implements ModuleFileSystem, Startable {
+public class DefaultModuleFileSystem implements ModuleFileSystem {
 
   private final String moduleKey;
   private final FileIndex index;
@@ -57,9 +57,12 @@ public class DefaultModuleFileSystem implements ModuleFileSystem, Startable {
   private List<File> sourceFiles = Lists.newArrayList();
   private List<File> testFiles = Lists.newArrayList();
   private AnalysisMode analysisMode;
-  private boolean dirsChanged = false;
+  private ComponentIndexer componentIndexer;
+  private boolean indexed;
 
-  public DefaultModuleFileSystem(Project module, Settings settings, FileIndex index, ModuleFileSystemInitializer initializer, AnalysisMode analysisMode) {
+  public DefaultModuleFileSystem(Project module, Settings settings, FileIndex index, ModuleFileSystemInitializer initializer, AnalysisMode analysisMode,
+    ComponentIndexer componentIndexer) {
+    this.componentIndexer = componentIndexer;
     this.moduleKey = module.getKey();
     this.settings = settings;
     this.index = index;
@@ -123,8 +126,7 @@ public class DefaultModuleFileSystem implements ModuleFileSystem, Startable {
    */
   @Deprecated
   void addSourceDir(File dir) {
-    sourceDirs.add(dir);
-    dirsChanged = true;
+    throw new UnsupportedOperationException("Modifications of the file system are not permitted");
   }
 
   /**
@@ -133,8 +135,7 @@ public class DefaultModuleFileSystem implements ModuleFileSystem, Startable {
    */
   @Deprecated
   void addTestDir(File dir) {
-    testDirs.add(dir);
-    dirsChanged = true;
+    throw new UnsupportedOperationException("Modifications of the file system are not permitted");
   }
 
   @Override
@@ -157,10 +158,6 @@ public class DefaultModuleFileSystem implements ModuleFileSystem, Startable {
    * @since 4.0
    */
   public Iterable<InputFile> inputFiles(FileQuery query) {
-    if (dirsChanged) {
-      index();
-      dirsChanged = false;
-    }
     List<InputFile> result = Lists.newArrayList();
     FileQueryFilter filter = new FileQueryFilter(analysisMode, query);
     for (InputFile input : index.inputFiles(moduleKey)) {
@@ -176,16 +173,6 @@ public class DefaultModuleFileSystem implements ModuleFileSystem, Startable {
     return InputFiles.toFiles(inputFiles(query));
   }
 
-  @Override
-  public void start() {
-    index();
-  }
-
-  @Override
-  public void stop() {
-    // nothing to do
-  }
-
   public void resetDirs(File basedir, File buildDir, List<File> sourceDirs, List<File> testDirs, List<File> binaryDirs) {
     Preconditions.checkNotNull(basedir, "Basedir can't be null");
     this.baseDir = basedir;
@@ -193,11 +180,15 @@ public class DefaultModuleFileSystem implements ModuleFileSystem, Startable {
     this.sourceDirs = existingDirs(sourceDirs);
     this.testDirs = existingDirs(testDirs);
     this.binaryDirs = existingDirs(binaryDirs);
-    index();
   }
 
   public void index() {
+    if (indexed) {
+      throw new SonarException("Module filesystem can only be indexed once");
+    }
+    indexed = true;
     index.index(this);
+    componentIndexer.execute(this);
   }
 
   private List<File> existingDirs(List<File> dirs) {
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileIndex.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileIndex.java
index d1ffbd92605..34d117be2ee 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileIndex.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileIndex.java
@@ -120,6 +120,7 @@ public class FileIndex implements BatchComponent {
     }
 
     logger.info(String.format("%d files indexed", progress.count));
+
   }
 
   private void indexFiles(DefaultModuleFileSystem fileSystem, Progress progress, List<File> sourceDirs, List<File> sourceFiles, String type) {
@@ -161,14 +162,9 @@ public class FileIndex implements BatchComponent {
 
   @CheckForNull
   private InputFile newInputFile(ModuleFileSystem fileSystem, File sourceDir, String type, File file, String path) {
-    String lang = languageRecognizer.of(file);
-    if (lang == null) {
-      return null;
-    }
 
     Map<String, String> attributes = Maps.newHashMap();
     set(attributes, InputFile.ATTRIBUTE_TYPE, type);
-    set(attributes, InputFile.ATTRIBUTE_LANGUAGE, lang);
 
     // paths
     set(attributes, InputFile.ATTRIBUTE_SOURCEDIR_PATH, PathUtils.canonicalPath(sourceDir));
@@ -177,16 +173,22 @@ public class FileIndex implements BatchComponent {
 
     String resourceKey = PathUtils.sanitize(path);
     set(attributes, DefaultInputFile.ATTRIBUTE_COMPONENT_KEY, project.getEffectiveKey() + ":" + resourceKey);
+    // hash + status
+    initStatus(file, fileSystem.sourceCharset(), path, attributes);
+
+    DefaultInputFile inputFile = DefaultInputFile.create(file, fileSystem.sourceCharset(), path, attributes);
+    String lang = languageRecognizer.of(inputFile);
+    if (lang == null) {
+      return null;
+    }
+    set(inputFile.attributes(), InputFile.ATTRIBUTE_LANGUAGE, lang);
     if (Java.KEY.equals(lang)) {
-      set(attributes, DefaultInputFile.ATTRIBUTE_COMPONENT_DEPRECATED_KEY, project.getEffectiveKey() + ":"
+      set(inputFile.attributes(), DefaultInputFile.ATTRIBUTE_COMPONENT_DEPRECATED_KEY, project.getEffectiveKey() + ":"
         + JavaFile.fromRelativePath(sourceRelativePath, false).getDeprecatedKey());
     } else {
-      set(attributes, DefaultInputFile.ATTRIBUTE_COMPONENT_DEPRECATED_KEY, project.getEffectiveKey() + ":" + sourceRelativePath);
+      set(inputFile.attributes(), DefaultInputFile.ATTRIBUTE_COMPONENT_DEPRECATED_KEY, project.getEffectiveKey() + ":" + sourceRelativePath);
     }
-    // hash + status
-    initStatus(file, fileSystem.sourceCharset(), path, attributes);
-
-    return DefaultInputFile.create(file, fileSystem.sourceCharset(), path, attributes);
+    return inputFile;
   }
 
   private void initStatus(File file, Charset charset, String baseRelativePath, Map<String, String> attributes) {
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/LanguageRecognizer.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/LanguageRecognizer.java
index fa8269afaad..26d898ff2ea 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/LanguageRecognizer.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/LanguageRecognizer.java
@@ -20,79 +20,126 @@
 package org.sonar.batch.scan.filesystem;
 
 import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Maps;
 import com.google.common.collect.SetMultimap;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.lang.StringUtils;
 import org.picocontainer.Startable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.sonar.api.BatchComponent;
+import org.sonar.api.CoreProperties;
+import org.sonar.api.config.Settings;
 import org.sonar.api.resources.Language;
-import org.sonar.api.resources.Project;
+import org.sonar.api.resources.Languages;
+import org.sonar.api.scan.filesystem.internal.InputFile;
+import org.sonar.api.utils.SonarException;
 
 import javax.annotation.CheckForNull;
-import java.io.File;
+
+import java.util.Map;
 import java.util.Set;
 
 /**
- * Detect language of source files. Simplistic, based on file extensions.
+ * Detect language of source files.
  */
 public class LanguageRecognizer implements BatchComponent, Startable {
 
-  private final Project project;
-  private final Language[] languages;
+  private static final Logger LOG = LoggerFactory.getLogger(LanguageRecognizer.class);
+
+  private final Languages languages;
 
   /**
    * Lower-case extension -> languages
    */
   private SetMultimap<String, String> langsByExtension = HashMultimap.create();
+  private Map<String, PathPattern[]> patternByLanguage = Maps.newLinkedHashMap();
 
-  /**
-   * Some plugins, like web and cobol, can analyze all the source files, whatever
-   * their file extension. This behavior is kept for backward-compatibility,
-   * but it should be fixed with future multi-language support.
-   */
-  private boolean ignoreFileExtension = false;
+  private Settings settings;
 
-  public LanguageRecognizer(Project project, Language[] languages) {
-    this.project = project;
+  public LanguageRecognizer(Settings settings, Languages languages) {
+    this.settings = settings;
     this.languages = languages;
   }
 
-  /**
-   * When no language plugin is installed
-   */
-  public LanguageRecognizer(Project project) {
-    this(project, new Language[0]);
-  }
-
   @Override
   public void start() {
-    for (Language language : languages) {
-      if (language.getFileSuffixes().length == 0 && language.getKey().equals(project.getLanguageKey())) {
-        ignoreFileExtension = true;
-
-      } else {
-        for (String suffix : language.getFileSuffixes()) {
-          String extension = sanitizeExtension(suffix);
-          langsByExtension.put(extension, language.getKey());
-        }
+    for (Language language : languages.all()) {
+      for (String suffix : language.getFileSuffixes()) {
+        String extension = sanitizeExtension(suffix);
+        langsByExtension.put(extension, language.getKey());
+      }
+      String[] filePatterns = settings.getStringArray(getFilePatternPropKey(language.getKey()));
+      PathPattern[] pathPatterns = PathPattern.create(filePatterns);
+      if (pathPatterns.length > 0) {
+        patternByLanguage.put(language.getKey(), pathPatterns);
       }
     }
   }
 
+  private String getFilePatternPropKey(String languageKey) {
+    return "sonar." + languageKey + ".filePatterns";
+  }
+
   @Override
   public void stop() {
     // do nothing
   }
 
   @CheckForNull
-  String of(File file) {
-    if (ignoreFileExtension) {
-      return project.getLanguageKey();
+  String of(InputFile inputFile) {
+    // First try with patterns
+    String forcedLanguage = null;
+    for (Map.Entry<String, PathPattern[]> languagePattern : patternByLanguage.entrySet()) {
+      PathPattern[] patterns = languagePattern.getValue();
+      for (PathPattern pathPattern : patterns) {
+        if (pathPattern.match(inputFile)) {
+          if (forcedLanguage == null) {
+            forcedLanguage = languagePattern.getKey();
+            break;
+          } else {
+            // Language was already forced by another pattern
+            throw new SonarException("Language of file '" + inputFile.path() + "' can not be decided as the file matches patterns of both " + getFilePatternPropKey(forcedLanguage)
+              + " and "
+              + getFilePatternPropKey(languagePattern.getKey()));
+          }
+        }
+      }
+    }
+    if (forcedLanguage != null) {
+      LOG.debug("Language of file '" + inputFile.path() + "' was forced to '" + forcedLanguage + "'");
+      return forcedLanguage;
+    }
+
+    String extension = sanitizeExtension(FilenameUtils.getExtension(inputFile.file().getName()));
+
+    // Check if deprecated sonar.language is used
+    String languageKey = settings.getString(CoreProperties.PROJECT_LANGUAGE_PROPERTY);
+    if (StringUtils.isNotBlank(languageKey)) {
+      Language language = languages.get(languageKey);
+      if (language == null) {
+        throw new SonarException("No language is installed with key '" + languageKey + "'. Please update property '" + CoreProperties.PROJECT_LANGUAGE_PROPERTY + "'");
+      }
+      // Languages without declared suffixes match everything
+      String[] fileSuffixes = language.getFileSuffixes();
+      if (fileSuffixes.length == 0) {
+        return languageKey;
+      }
+      for (String fileSuffix : fileSuffixes) {
+        if (sanitizeExtension(fileSuffix).equals(extension)) {
+          return languageKey;
+        }
+      }
+      return null;
     }
-    // multi-language is not supported yet. Filter on project language
-    String extension = sanitizeExtension(FilenameUtils.getExtension(file.getName()));
+
+    // At this point use extension to detect language
     Set<String> langs = langsByExtension.get(extension);
-    return langs.contains(project.getLanguageKey()) ? project.getLanguageKey() : null;
+    if (langs.size() > 1) {
+      throw new SonarException("Language of file '" + inputFile.path() + "' can not be decided as the file extension '" + extension + "' is declared by several languages: "
+        + StringUtils.join(langs, ", "));
+    }
+    return langs.isEmpty() ? null : langs.iterator().next();
   }
 
   static String sanitizeExtension(String suffix) {
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/PathPattern.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/PathPattern.java
index 446052c84c5..650f447e476 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/PathPattern.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/PathPattern.java
@@ -81,7 +81,7 @@ abstract class PathPattern {
   }
 
   /**
-   * Path relative to source directory
+   * Path relative to module basedir
    */
   private static class RelativePathPattern extends PathPattern {
     private RelativePathPattern(String pattern) {
@@ -90,7 +90,7 @@ abstract class PathPattern {
 
     @Override
     boolean match(InputFile inputFile) {
-      String path = inputFile.attribute(InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH);
+      String path = inputFile.path();
       return path != null && pattern.match(path);
     }