SONAR-5077 Add option to import all files even if no language

9 files changed, 86 insertions, 33 deletions

diff --git a/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/CorePlugin.java b/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/CorePlugin.java
index cd9f54ee857..167e7edf658 100644
--- a/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/CorePlugin.java
+++ b/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/CorePlugin.java
@@ -270,6 +270,16 @@ import java.util.List;
     project = true,
     global = false,
     category = CoreProperties.CATEGORY_SCM
+  ),
+  @Property(
+    key = CoreProperties.INDEX_ALL_FILES_KEY,
+    defaultValue = "false",
+    name = "Index all files",
+    description = "Should all files be indexed even if there is no matching language plugin installed.",
+    module = false,
+    project = true,
+    global = true,
+    category = CoreProperties.CATEGORY_GENERAL
   )
 })
 public final class CorePlugin extends SonarPlugin {
diff --git a/sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java b/sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java
index e83fb0b8962..88f6ac854c4 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java
@@ -135,29 +135,33 @@ public class SourcePersister implements ScanPersister {
     String newData = getSourceData(inputFile);
     String newDataHash = newData != null ? DigestUtils.md5Hex(newData) : "0";
     Date now = system2.newDate();
-    if (previous == null) {
-      FileSourceDto newFileSource = new FileSourceDto()
-        .setProjectUuid(projectTree.getRootProject().getUuid())
-        .setFileUuid(fileUuid)
-        .setData(newData)
-        .setDataHash(newDataHash)
-        .setSrcHash(inputFile.hash())
-        .setLineHashes(lineHashesAsMd5Hex(inputFile))
-        .setCreatedAt(now.getTime())
-        .setUpdatedAt(now.getTime());
-      mapper.insert(newFileSource);
-      session.commit();
-    } else {
-      if (!newDataHash.equals(previous.getDataHash())) {
-        previous
+    try {
+      if (previous == null) {
+        FileSourceDto newFileSource = new FileSourceDto()
+          .setProjectUuid(projectTree.getRootProject().getUuid())
+          .setFileUuid(fileUuid)
           .setData(newData)
-          .setLineHashes(lineHashesAsMd5Hex(inputFile))
           .setDataHash(newDataHash)
           .setSrcHash(inputFile.hash())
+          .setLineHashes(lineHashesAsMd5Hex(inputFile))
+          .setCreatedAt(now.getTime())
           .setUpdatedAt(now.getTime());
-        mapper.update(previous);
+        mapper.insert(newFileSource);
         session.commit();
+      } else {
+        if (!newDataHash.equals(previous.getDataHash())) {
+          previous
+            .setData(newData)
+            .setLineHashes(lineHashesAsMd5Hex(inputFile))
+            .setDataHash(newDataHash)
+            .setSrcHash(inputFile.hash())
+            .setUpdatedAt(now.getTime());
+          mapper.update(previous);
+          session.commit();
+        }
       }
+    } catch (Exception e) {
+      throw new IllegalStateException("Unable to save file sources for " + inputPath.absolutePath(), e);
     }
   }
 
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilder.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilder.java
index 8b00b2640a4..49303a056fd 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilder.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilder.java
@@ -22,9 +22,11 @@ package org.sonar.batch.scan.filesystem;
 import org.apache.commons.io.FilenameUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.sonar.api.CoreProperties;
 import org.sonar.api.batch.fs.FileSystem;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.internal.DeprecatedDefaultInputFile;
+import org.sonar.api.config.Settings;
 import org.sonar.api.scan.filesystem.PathResolver;
 import org.sonar.batch.bootstrap.AnalysisMode;
 import org.sonar.batch.util.DeprecatedKeyUtils;
@@ -44,15 +46,17 @@ class InputFileBuilder {
   private final StatusDetection statusDetection;
   private final DefaultModuleFileSystem fs;
   private final AnalysisMode analysisMode;
+  private final Settings settings;
 
   InputFileBuilder(String moduleKey, PathResolver pathResolver, LanguageDetection langDetection,
-    StatusDetection statusDetection, DefaultModuleFileSystem fs, AnalysisMode analysisMode) {
+    StatusDetection statusDetection, DefaultModuleFileSystem fs, AnalysisMode analysisMode, Settings settings) {
     this.moduleKey = moduleKey;
     this.pathResolver = pathResolver;
     this.langDetection = langDetection;
     this.statusDetection = statusDetection;
     this.fs = fs;
     this.analysisMode = analysisMode;
+    this.settings = settings;
   }
 
   String moduleKey() {
@@ -98,7 +102,7 @@ class InputFileBuilder {
     inputFile.setEncoding(fs.encoding().name());
 
     String lang = langDetection.language(inputFile);
-    if (lang == null) {
+    if (lang == null && !settings.getBoolean(CoreProperties.INDEX_ALL_FILES_KEY)) {
       return null;
     }
     inputFile.setLanguage(lang);
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilderFactory.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilderFactory.java
index 59f1e667edd..ac8c79db2ab 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilderFactory.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilderFactory.java
@@ -21,6 +21,7 @@ package org.sonar.batch.scan.filesystem;
 
 import org.sonar.api.BatchComponent;
 import org.sonar.api.batch.bootstrap.ProjectDefinition;
+import org.sonar.api.config.Settings;
 import org.sonar.api.scan.filesystem.PathResolver;
 import org.sonar.batch.bootstrap.AnalysisMode;
 
@@ -31,22 +32,24 @@ public class InputFileBuilderFactory implements BatchComponent {
   private final LanguageDetectionFactory langDetectionFactory;
   private final StatusDetectionFactory statusDetectionFactory;
   private final AnalysisMode analysisMode;
+  private final Settings settings;
 
   public InputFileBuilderFactory(ProjectDefinition def, PathResolver pathResolver, LanguageDetectionFactory langDetectionFactory,
-    StatusDetectionFactory statusDetectionFactory, AnalysisMode analysisMode) {
-    this(def.getKeyWithBranch(), pathResolver, langDetectionFactory, statusDetectionFactory, analysisMode);
+    StatusDetectionFactory statusDetectionFactory, AnalysisMode analysisMode, Settings settings) {
+    this(def.getKeyWithBranch(), pathResolver, langDetectionFactory, statusDetectionFactory, analysisMode, settings);
   }
 
   private InputFileBuilderFactory(String effectiveKey, PathResolver pathResolver, LanguageDetectionFactory langDetectionFactory,
-    StatusDetectionFactory statusDetectionFactory, AnalysisMode analysisMode) {
+    StatusDetectionFactory statusDetectionFactory, AnalysisMode analysisMode, Settings settings) {
     this.moduleKey = effectiveKey;
     this.pathResolver = pathResolver;
     this.langDetectionFactory = langDetectionFactory;
     this.statusDetectionFactory = statusDetectionFactory;
     this.analysisMode = analysisMode;
+    this.settings = settings;
   }
 
   InputFileBuilder create(DefaultModuleFileSystem fs) {
-    return new InputFileBuilder(moduleKey, pathResolver, langDetectionFactory.create(), statusDetectionFactory.create(), fs, analysisMode);
+    return new InputFileBuilder(moduleKey, pathResolver, langDetectionFactory.create(), statusDetectionFactory.create(), fs, analysisMode, settings);
   }
 }
diff --git a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/fs/FileSystemMediumTest.java b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/fs/FileSystemMediumTest.java
index 9dd41b52076..893bc1a132a 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/fs/FileSystemMediumTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/fs/FileSystemMediumTest.java
@@ -19,8 +19,6 @@
  */
 package org.sonar.batch.mediumtest.fs;
 
-import org.sonar.batch.mediumtest.TaskResult;
-
 import com.google.common.collect.ImmutableMap;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.StringUtils;
@@ -34,6 +32,7 @@ import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.utils.MessageException;
 import org.sonar.api.utils.System2;
 import org.sonar.batch.mediumtest.BatchMediumTester;
+import org.sonar.batch.mediumtest.TaskResult;
 import org.sonar.xoo.XooPlugin;
 
 import java.io.File;
@@ -205,4 +204,28 @@ public class FileSystemMediumTest {
     }
   }
 
+  @Test
+  public void indexAnyFile() throws IOException {
+    File srcDir = new File(baseDir, "src");
+    srcDir.mkdir();
+
+    File xooFile = new File(srcDir, "sample.xoo");
+    FileUtils.write(xooFile, "Sample xoo\ncontent");
+
+    File otherFile = new File(srcDir, "sample.other");
+    FileUtils.write(otherFile, "Sample other\ncontent");
+
+    TaskResult result = tester.newTask()
+      .properties(builder
+        .put("sonar.sources", "src")
+        .put("sonar.index_all_files", "true")
+        .build())
+      .start();
+
+    assertThat(result.inputFiles()).hasSize(2);
+    assertThat(result.inputFile("src/sample.other").type()).isEqualTo(InputFile.Type.MAIN);
+    assertThat(result.inputFile("src/sample.other").relativePath()).isEqualTo("src/sample.other");
+    assertThat(result.inputFile("src/sample.other").language()).isNull();
+  }
+
 }
diff --git a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InputFileBuilderFactoryTest.java b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InputFileBuilderFactoryTest.java
index 48941e1e208..9a753c7e66f 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InputFileBuilderFactoryTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InputFileBuilderFactoryTest.java
@@ -22,6 +22,7 @@ package org.sonar.batch.scan.filesystem;
 import org.junit.Test;
 import org.mockito.Mockito;
 import org.sonar.api.batch.bootstrap.ProjectDefinition;
+import org.sonar.api.config.Settings;
 import org.sonar.api.scan.filesystem.PathResolver;
 import org.sonar.batch.bootstrap.AnalysisMode;
 
@@ -38,7 +39,7 @@ public class InputFileBuilderFactoryTest {
     AnalysisMode analysisMode = mock(AnalysisMode.class);
 
     InputFileBuilderFactory factory = new InputFileBuilderFactory(ProjectDefinition.create().setKey("struts"), pathResolver, langDetectionFactory,
-      statusDetectionFactory, analysisMode);
+      statusDetectionFactory, analysisMode, new Settings());
     InputFileBuilder builder = factory.create(fs);
 
     assertThat(builder.langDetection()).isNotNull();
diff --git a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InputFileBuilderTest.java b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InputFileBuilderTest.java
index cf2a1cbe22b..3d1eb9b050d 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InputFileBuilderTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InputFileBuilderTest.java
@@ -26,6 +26,7 @@ import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.internal.DeprecatedDefaultInputFile;
+import org.sonar.api.config.Settings;
 import org.sonar.api.scan.filesystem.PathResolver;
 import org.sonar.api.utils.PathUtils;
 import org.sonar.batch.bootstrap.AnalysisMode;
@@ -66,7 +67,7 @@ public class InputFileBuilderTest {
       .thenReturn(InputFile.Status.ADDED);
 
     InputFileBuilder builder = new InputFileBuilder("struts", new PathResolver(),
-      langDetection, statusDetection, fs, analysisMode);
+      langDetection, statusDetection, fs, analysisMode, new Settings());
     DeprecatedDefaultInputFile inputFile = builder.create(srcFile);
     inputFile = builder.complete(inputFile, InputFile.Type.MAIN);
 
@@ -92,7 +93,7 @@ public class InputFileBuilderTest {
     when(fs.baseDir()).thenReturn(basedir);
 
     InputFileBuilder builder = new InputFileBuilder("struts", new PathResolver(),
-      langDetection, statusDetection, fs, analysisMode);
+      langDetection, statusDetection, fs, analysisMode, new Settings());
     DeprecatedDefaultInputFile inputFile = builder.create(srcFile);
 
     assertThat(inputFile).isNull();
@@ -112,7 +113,7 @@ public class InputFileBuilderTest {
     when(langDetection.language(any(InputFile.class))).thenReturn(null);
 
     InputFileBuilder builder = new InputFileBuilder("struts", new PathResolver(),
-      langDetection, statusDetection, fs, analysisMode);
+      langDetection, statusDetection, fs, analysisMode, new Settings());
     DeprecatedDefaultInputFile inputFile = builder.create(srcFile);
     inputFile = builder.complete(inputFile, InputFile.Type.MAIN);
 
@@ -139,7 +140,7 @@ public class InputFileBuilderTest {
       .thenReturn(InputFile.Status.ADDED);
 
     InputFileBuilder builder = new InputFileBuilder("struts", new PathResolver(),
-      langDetection, statusDetection, fs, analysisMode);
+      langDetection, statusDetection, fs, analysisMode, new Settings());
     DeprecatedDefaultInputFile inputFile = builder.create(srcFile);
     inputFile = builder.complete(inputFile, InputFile.Type.MAIN);
 
@@ -168,7 +169,7 @@ public class InputFileBuilderTest {
       .thenReturn(InputFile.Status.ADDED);
 
     InputFileBuilder builder = new InputFileBuilder("struts", new PathResolver(),
-      langDetection, statusDetection, fs, analysisMode);
+      langDetection, statusDetection, fs, analysisMode, new Settings());
     DeprecatedDefaultInputFile inputFile = builder.create(srcFile);
     inputFile = builder.complete(inputFile, InputFile.Type.MAIN);
 
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/CoreProperties.java b/sonar-plugin-api/src/main/java/org/sonar/api/CoreProperties.java
index 38626a5c126..30e727324ad 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/CoreProperties.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/CoreProperties.java
@@ -552,4 +552,9 @@ public interface CoreProperties {
    * @since 5.0
    */
   String SCM_PROVIDER_KEY = "sonar.scm.provider";
+
+  /**
+   * @since 5.1
+   */
+  String INDEX_ALL_FILES_KEY = "sonar.index_all_files";
 }
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/InputFile.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/InputFile.java
index 357a7c99103..2cf8ee33c37 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/InputFile.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/InputFile.java
@@ -19,6 +19,8 @@
  */
 package org.sonar.api.batch.fs;
 
+import javax.annotation.CheckForNull;
+
 import java.io.File;
 
 /**
@@ -70,9 +72,9 @@ public interface InputFile extends InputPath {
   File file();
 
   /**
-   * Language, for example "java" or "php". It's automatically guessed if it is not
-   * set in project settings.
+   * Language, for example "java" or "php". Can be null if indexation of all files is enabled and no language claims to support the file.
    */
+  @CheckForNull
   String language();
 
   /**