SONAR-5077 Add option to import all files even if no language

6 files changed, 67 insertions, 31 deletions

diff --git a/sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java b/sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java
index e83fb0b8962..88f6ac854c4 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java
@@ -135,29 +135,33 @@ public class SourcePersister implements ScanPersister {
     String newData = getSourceData(inputFile);
     String newDataHash = newData != null ? DigestUtils.md5Hex(newData) : "0";
     Date now = system2.newDate();
-    if (previous == null) {
-      FileSourceDto newFileSource = new FileSourceDto()
-        .setProjectUuid(projectTree.getRootProject().getUuid())
-        .setFileUuid(fileUuid)
-        .setData(newData)
-        .setDataHash(newDataHash)
-        .setSrcHash(inputFile.hash())
-        .setLineHashes(lineHashesAsMd5Hex(inputFile))
-        .setCreatedAt(now.getTime())
-        .setUpdatedAt(now.getTime());
-      mapper.insert(newFileSource);
-      session.commit();
-    } else {
-      if (!newDataHash.equals(previous.getDataHash())) {
-        previous
+    try {
+      if (previous == null) {
+        FileSourceDto newFileSource = new FileSourceDto()
+          .setProjectUuid(projectTree.getRootProject().getUuid())
+          .setFileUuid(fileUuid)
           .setData(newData)
-          .setLineHashes(lineHashesAsMd5Hex(inputFile))
           .setDataHash(newDataHash)
           .setSrcHash(inputFile.hash())
+          .setLineHashes(lineHashesAsMd5Hex(inputFile))
+          .setCreatedAt(now.getTime())
           .setUpdatedAt(now.getTime());
-        mapper.update(previous);
+        mapper.insert(newFileSource);
         session.commit();
+      } else {
+        if (!newDataHash.equals(previous.getDataHash())) {
+          previous
+            .setData(newData)
+            .setLineHashes(lineHashesAsMd5Hex(inputFile))
+            .setDataHash(newDataHash)
+            .setSrcHash(inputFile.hash())
+            .setUpdatedAt(now.getTime());
+          mapper.update(previous);
+          session.commit();
+        }
       }
+    } catch (Exception e) {
+      throw new IllegalStateException("Unable to save file sources for " + inputPath.absolutePath(), e);
     }
   }
 
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilder.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilder.java
index 8b00b2640a4..49303a056fd 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilder.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilder.java
@@ -22,9 +22,11 @@ package org.sonar.batch.scan.filesystem;
 import org.apache.commons.io.FilenameUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.sonar.api.CoreProperties;
 import org.sonar.api.batch.fs.FileSystem;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.internal.DeprecatedDefaultInputFile;
+import org.sonar.api.config.Settings;
 import org.sonar.api.scan.filesystem.PathResolver;
 import org.sonar.batch.bootstrap.AnalysisMode;
 import org.sonar.batch.util.DeprecatedKeyUtils;
@@ -44,15 +46,17 @@ class InputFileBuilder {
   private final StatusDetection statusDetection;
   private final DefaultModuleFileSystem fs;
   private final AnalysisMode analysisMode;
+  private final Settings settings;
 
   InputFileBuilder(String moduleKey, PathResolver pathResolver, LanguageDetection langDetection,
-    StatusDetection statusDetection, DefaultModuleFileSystem fs, AnalysisMode analysisMode) {
+    StatusDetection statusDetection, DefaultModuleFileSystem fs, AnalysisMode analysisMode, Settings settings) {
     this.moduleKey = moduleKey;
     this.pathResolver = pathResolver;
     this.langDetection = langDetection;
     this.statusDetection = statusDetection;
     this.fs = fs;
     this.analysisMode = analysisMode;
+    this.settings = settings;
   }
 
   String moduleKey() {
@@ -98,7 +102,7 @@ class InputFileBuilder {
     inputFile.setEncoding(fs.encoding().name());
 
     String lang = langDetection.language(inputFile);
-    if (lang == null) {
+    if (lang == null && !settings.getBoolean(CoreProperties.INDEX_ALL_FILES_KEY)) {
       return null;
     }
     inputFile.setLanguage(lang);
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilderFactory.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilderFactory.java
index 59f1e667edd..ac8c79db2ab 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilderFactory.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/InputFileBuilderFactory.java
@@ -21,6 +21,7 @@ package org.sonar.batch.scan.filesystem;
 
 import org.sonar.api.BatchComponent;
 import org.sonar.api.batch.bootstrap.ProjectDefinition;
+import org.sonar.api.config.Settings;
 import org.sonar.api.scan.filesystem.PathResolver;
 import org.sonar.batch.bootstrap.AnalysisMode;
 
@@ -31,22 +32,24 @@ public class InputFileBuilderFactory implements BatchComponent {
   private final LanguageDetectionFactory langDetectionFactory;
   private final StatusDetectionFactory statusDetectionFactory;
   private final AnalysisMode analysisMode;
+  private final Settings settings;
 
   public InputFileBuilderFactory(ProjectDefinition def, PathResolver pathResolver, LanguageDetectionFactory langDetectionFactory,
-    StatusDetectionFactory statusDetectionFactory, AnalysisMode analysisMode) {
-    this(def.getKeyWithBranch(), pathResolver, langDetectionFactory, statusDetectionFactory, analysisMode);
+    StatusDetectionFactory statusDetectionFactory, AnalysisMode analysisMode, Settings settings) {
+    this(def.getKeyWithBranch(), pathResolver, langDetectionFactory, statusDetectionFactory, analysisMode, settings);
   }
 
   private InputFileBuilderFactory(String effectiveKey, PathResolver pathResolver, LanguageDetectionFactory langDetectionFactory,
-    StatusDetectionFactory statusDetectionFactory, AnalysisMode analysisMode) {
+    StatusDetectionFactory statusDetectionFactory, AnalysisMode analysisMode, Settings settings) {
     this.moduleKey = effectiveKey;
     this.pathResolver = pathResolver;
     this.langDetectionFactory = langDetectionFactory;
     this.statusDetectionFactory = statusDetectionFactory;
     this.analysisMode = analysisMode;
+    this.settings = settings;
   }
 
   InputFileBuilder create(DefaultModuleFileSystem fs) {
-    return new InputFileBuilder(moduleKey, pathResolver, langDetectionFactory.create(), statusDetectionFactory.create(), fs, analysisMode);
+    return new InputFileBuilder(moduleKey, pathResolver, langDetectionFactory.create(), statusDetectionFactory.create(), fs, analysisMode, settings);
   }
 }
diff --git a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/fs/FileSystemMediumTest.java b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/fs/FileSystemMediumTest.java
index 9dd41b52076..893bc1a132a 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/fs/FileSystemMediumTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/fs/FileSystemMediumTest.java
@@ -19,8 +19,6 @@
  */
 package org.sonar.batch.mediumtest.fs;
 
-import org.sonar.batch.mediumtest.TaskResult;
-
 import com.google.common.collect.ImmutableMap;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.StringUtils;
@@ -34,6 +32,7 @@ import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.utils.MessageException;
 import org.sonar.api.utils.System2;
 import org.sonar.batch.mediumtest.BatchMediumTester;
+import org.sonar.batch.mediumtest.TaskResult;
 import org.sonar.xoo.XooPlugin;
 
 import java.io.File;
@@ -205,4 +204,28 @@ public class FileSystemMediumTest {
     }
   }
 
+  @Test
+  public void indexAnyFile() throws IOException {
+    File srcDir = new File(baseDir, "src");
+    srcDir.mkdir();
+
+    File xooFile = new File(srcDir, "sample.xoo");
+    FileUtils.write(xooFile, "Sample xoo\ncontent");
+
+    File otherFile = new File(srcDir, "sample.other");
+    FileUtils.write(otherFile, "Sample other\ncontent");
+
+    TaskResult result = tester.newTask()
+      .properties(builder
+        .put("sonar.sources", "src")
+        .put("sonar.index_all_files", "true")
+        .build())
+      .start();
+
+    assertThat(result.inputFiles()).hasSize(2);
+    assertThat(result.inputFile("src/sample.other").type()).isEqualTo(InputFile.Type.MAIN);
+    assertThat(result.inputFile("src/sample.other").relativePath()).isEqualTo("src/sample.other");
+    assertThat(result.inputFile("src/sample.other").language()).isNull();
+  }
+
 }
diff --git a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InputFileBuilderFactoryTest.java b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InputFileBuilderFactoryTest.java
index 48941e1e208..9a753c7e66f 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InputFileBuilderFactoryTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InputFileBuilderFactoryTest.java
@@ -22,6 +22,7 @@ package org.sonar.batch.scan.filesystem;
 import org.junit.Test;
 import org.mockito.Mockito;
 import org.sonar.api.batch.bootstrap.ProjectDefinition;
+import org.sonar.api.config.Settings;
 import org.sonar.api.scan.filesystem.PathResolver;
 import org.sonar.batch.bootstrap.AnalysisMode;
 
@@ -38,7 +39,7 @@ public class InputFileBuilderFactoryTest {
     AnalysisMode analysisMode = mock(AnalysisMode.class);
 
     InputFileBuilderFactory factory = new InputFileBuilderFactory(ProjectDefinition.create().setKey("struts"), pathResolver, langDetectionFactory,
-      statusDetectionFactory, analysisMode);
+      statusDetectionFactory, analysisMode, new Settings());
     InputFileBuilder builder = factory.create(fs);
 
     assertThat(builder.langDetection()).isNotNull();
diff --git a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InputFileBuilderTest.java b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InputFileBuilderTest.java
index cf2a1cbe22b..3d1eb9b050d 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InputFileBuilderTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InputFileBuilderTest.java
@@ -26,6 +26,7 @@ import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.internal.DeprecatedDefaultInputFile;
+import org.sonar.api.config.Settings;
 import org.sonar.api.scan.filesystem.PathResolver;
 import org.sonar.api.utils.PathUtils;
 import org.sonar.batch.bootstrap.AnalysisMode;
@@ -66,7 +67,7 @@ public class InputFileBuilderTest {
       .thenReturn(InputFile.Status.ADDED);
 
     InputFileBuilder builder = new InputFileBuilder("struts", new PathResolver(),
-      langDetection, statusDetection, fs, analysisMode);
+      langDetection, statusDetection, fs, analysisMode, new Settings());
     DeprecatedDefaultInputFile inputFile = builder.create(srcFile);
     inputFile = builder.complete(inputFile, InputFile.Type.MAIN);
 
@@ -92,7 +93,7 @@ public class InputFileBuilderTest {
     when(fs.baseDir()).thenReturn(basedir);
 
     InputFileBuilder builder = new InputFileBuilder("struts", new PathResolver(),
-      langDetection, statusDetection, fs, analysisMode);
+      langDetection, statusDetection, fs, analysisMode, new Settings());
     DeprecatedDefaultInputFile inputFile = builder.create(srcFile);
 
     assertThat(inputFile).isNull();
@@ -112,7 +113,7 @@ public class InputFileBuilderTest {
     when(langDetection.language(any(InputFile.class))).thenReturn(null);
 
     InputFileBuilder builder = new InputFileBuilder("struts", new PathResolver(),
-      langDetection, statusDetection, fs, analysisMode);
+      langDetection, statusDetection, fs, analysisMode, new Settings());
     DeprecatedDefaultInputFile inputFile = builder.create(srcFile);
     inputFile = builder.complete(inputFile, InputFile.Type.MAIN);
 
@@ -139,7 +140,7 @@ public class InputFileBuilderTest {
       .thenReturn(InputFile.Status.ADDED);
 
     InputFileBuilder builder = new InputFileBuilder("struts", new PathResolver(),
-      langDetection, statusDetection, fs, analysisMode);
+      langDetection, statusDetection, fs, analysisMode, new Settings());
     DeprecatedDefaultInputFile inputFile = builder.create(srcFile);
     inputFile = builder.complete(inputFile, InputFile.Type.MAIN);
 
@@ -168,7 +169,7 @@ public class InputFileBuilderTest {
       .thenReturn(InputFile.Status.ADDED);
 
     InputFileBuilder builder = new InputFileBuilder("struts", new PathResolver(),
-      langDetection, statusDetection, fs, analysisMode);
+      langDetection, statusDetection, fs, analysisMode, new Settings());
     DeprecatedDefaultInputFile inputFile = builder.create(srcFile);
     inputFile = builder.complete(inputFile, InputFile.Type.MAIN);