SONAR-4783 API - drop the extension point SourceImporter

14 files changed, 285 insertions, 96 deletions

diff --git a/sonar-batch/src/main/java/org/sonar/batch/DefaultSensorContext.java b/sonar-batch/src/main/java/org/sonar/batch/DefaultSensorContext.java
index bc3c0424451..4ca86170d38 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/DefaultSensorContext.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/DefaultSensorContext.java
@@ -19,8 +19,6 @@
  */
 package org.sonar.batch;
 
-import org.sonar.core.measure.MeasurementFilters;
-
 import org.sonar.api.batch.Event;
 import org.sonar.api.batch.SensorContext;
 import org.sonar.api.batch.SonarIndex;
@@ -32,6 +30,7 @@ import org.sonar.api.resources.Project;
 import org.sonar.api.resources.ProjectLink;
 import org.sonar.api.resources.Resource;
 import org.sonar.api.rules.Violation;
+import org.sonar.core.measure.MeasurementFilters;
 
 import java.util.Collection;
 import java.util.Date;
@@ -55,11 +54,11 @@ public class DefaultSensorContext implements SensorContext {
   }
 
   public boolean index(Resource resource) {
-    return index.index(resource);
+    return true;
   }
 
   public boolean index(Resource resource, Resource parentReference) {
-    return index.index(resource, parentReference);
+    return true;
   }
 
   public boolean isExcluded(Resource reference) {
@@ -123,7 +122,7 @@ public class DefaultSensorContext implements SensorContext {
   }
 
   public Measure saveMeasure(Resource resource, Measure measure) {
-    if(filters.accept(resource, measure)) {
+    if (filters.accept(resource, measure)) {
       return index.addMeasure(resourceOrProject(resource), measure);
     } else {
       return measure;
@@ -166,7 +165,6 @@ public class DefaultSensorContext implements SensorContext {
   }
 
   public void saveSource(Resource reference, String source) {
-    index.setSource(reference, source);
   }
 
   public void saveLink(ProjectLink link) {
@@ -190,6 +188,6 @@ public class DefaultSensorContext implements SensorContext {
   }
 
   private Resource resourceOrProject(Resource resource) {
-    return resource!=null ? resource : project;
+    return resource != null ? resource : project;
   }
 }
diff --git a/sonar-batch/src/main/java/org/sonar/batch/phases/FileIndexer.java b/sonar-batch/src/main/java/org/sonar/batch/phases/FileIndexer.java
index dc1e8eda2a7..025f1effc45 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/phases/FileIndexer.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/phases/FileIndexer.java
@@ -19,9 +19,13 @@
  */
 package org.sonar.batch.phases;
 
+import com.google.common.base.CharMatcher;
+import com.google.common.io.Files;
 import org.sonar.api.BatchComponent;
+import org.sonar.api.CoreProperties;
 import org.sonar.api.batch.InstantiationStrategy;
-import org.sonar.api.batch.SensorContext;
+import org.sonar.api.batch.SonarIndex;
+import org.sonar.api.config.Settings;
 import org.sonar.api.resources.File;
 import org.sonar.api.resources.Java;
 import org.sonar.api.resources.JavaFile;
@@ -31,6 +35,7 @@ import org.sonar.api.resources.Qualifiers;
 import org.sonar.api.resources.Resource;
 import org.sonar.api.scan.filesystem.FileQuery;
 import org.sonar.api.scan.filesystem.internal.InputFile;
+import org.sonar.api.utils.SonarException;
 import org.sonar.batch.scan.filesystem.DefaultModuleFileSystem;
 
 /**
@@ -40,24 +45,30 @@ import org.sonar.batch.scan.filesystem.DefaultModuleFileSystem;
 @InstantiationStrategy(InstantiationStrategy.PER_PROJECT)
 public class FileIndexer implements BatchComponent {
 
-  private Project module;
-  private DefaultModuleFileSystem fs;
+  private final Project module;
+  private final DefaultModuleFileSystem fs;
+  private final Languages languages;
+  private final Settings settings;
+  private final SonarIndex sonarIndex;
 
-  private Languages languages;
+  private boolean importSource;
 
-  public FileIndexer(Project module, DefaultModuleFileSystem fs, Languages languages) {
+  public FileIndexer(Project module, DefaultModuleFileSystem fs, Languages languages, SonarIndex sonarIndex, Settings settings) {
     this.module = module;
     this.fs = fs;
     this.languages = languages;
+    this.sonarIndex = sonarIndex;
+    this.settings = settings;
   }
 
-  public void execute(SensorContext context) {
+  public void execute() {
+    this.importSource = settings.getBoolean(CoreProperties.CORE_IMPORT_SOURCES_PROPERTY);
     String languageKey = module.getLanguageKey();
-    indexFiles(fs.inputFiles(FileQuery.onSource().onLanguage(languageKey)), false, context, languageKey);
-    indexFiles(fs.inputFiles(FileQuery.onTest().onLanguage(languageKey)), true, context, languageKey);
+    indexFiles(fs.inputFiles(FileQuery.onSource().onLanguage(languageKey)), false, languageKey);
+    indexFiles(fs.inputFiles(FileQuery.onTest().onLanguage(languageKey)), true, languageKey);
   }
 
-  private void indexFiles(Iterable<InputFile> files, boolean unitTest, SensorContext context, String languageKey) {
+  private void indexFiles(Iterable<InputFile> files, boolean unitTest, String languageKey) {
     for (InputFile inputFile : files) {
       Resource sonarFile;
       if (Java.KEY.equals(languageKey)) {
@@ -71,7 +82,18 @@ public class FileIndexer implements BatchComponent {
       }
       if (sonarFile != null) {
         sonarFile.setPath(inputFile.path());
-        context.index(sonarFile);
+        sonarIndex.index(sonarFile);
+        try {
+          if (importSource) {
+            String source = Files.toString(inputFile.file(), inputFile.encoding());
+            // SONAR-3860 Remove BOM character from source
+            source = CharMatcher.anyOf("\uFEFF").removeFrom(source);
+            sonarIndex.setSource(sonarFile, source);
+          }
+        } catch (Exception e) {
+          throw new SonarException("Unable to read and import the source file : '" + inputFile.absolutePath() + "' with the charset : '"
+            + inputFile.encoding() + "'.", e);
+        }
       }
     }
   }
diff --git a/sonar-batch/src/main/java/org/sonar/batch/phases/SensorsExecutor.java b/sonar-batch/src/main/java/org/sonar/batch/phases/SensorsExecutor.java
index f337007fe7f..d66e08ad0a7 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/phases/SensorsExecutor.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/phases/SensorsExecutor.java
@@ -65,7 +65,7 @@ public class SensorsExecutor implements BatchComponent {
     Collection<Sensor> sensors = selector.select(Sensor.class, project, true, sensorMatcher);
     eventBus.fireEvent(new SensorsPhaseEvent(Lists.newArrayList(sensors), true));
 
-    fileIndexer.execute(context);
+    fileIndexer.execute();
 
     for (Sensor sensor : sensors) {
       // SONAR-2965 In case the sensor takes too much time we close the session to not face a timeout
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileIndex.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileIndex.java
index e58052abd99..19826e193b7 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileIndex.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileIndex.java
@@ -40,6 +40,7 @@ import org.sonar.api.utils.PathUtils;
 
 import javax.annotation.CheckForNull;
 import javax.annotation.Nullable;
+
 import java.io.File;
 import java.nio.charset.Charset;
 import java.util.Collection;
@@ -77,7 +78,7 @@ public class FileIndex implements BatchComponent {
   private final Project project;
 
   public FileIndex(List<InputFileFilter> filters, LanguageRecognizer languageRecognizer,
-                   InputFileCache cache, FileHashes fileHashes, PathResolver pathResolver, Project project) {
+    InputFileCache cache, FileHashes fileHashes, PathResolver pathResolver, Project project) {
     this.filters = filters;
     this.languageRecognizer = languageRecognizer;
     this.cache = cache;
@@ -127,7 +128,7 @@ public class FileIndex implements BatchComponent {
       if (sourceDirPath == null) {
         LoggerFactory.getLogger(getClass()).warn(String.format(
           "File '%s' is not declared in source directories %s", sourceFile.getAbsoluteFile(), StringUtils.join(sourceDirs, ", ")
-        ));
+          ));
       } else {
         indexFile(fileSystem, progress, sourceDirPath.dir(), sourceFile, type);
       }
@@ -183,7 +184,7 @@ public class FileIndex implements BatchComponent {
     // hash + status
     initStatus(file, fileSystem.sourceCharset(), path, attributes);
 
-    return DefaultInputFile.create(file, path, attributes);
+    return DefaultInputFile.create(file, fileSystem.sourceCharset(), path, attributes);
   }
 
   private void initStatus(File file, Charset charset, String baseRelativePath, Map<String, String> attributes) {
diff --git a/sonar-batch/src/test/java/org/sonar/batch/phases/FileIndexerTest.java b/sonar-batch/src/test/java/org/sonar/batch/phases/FileIndexerTest.java
index 09282526597..d961d85f856 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/phases/FileIndexerTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/phases/FileIndexerTest.java
@@ -19,15 +19,26 @@
  */
 package org.sonar.batch.phases;
 
+import com.google.common.base.Charsets;
+import edu.emory.mathcs.backport.java.util.Collections;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.CharEncoding;
+import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
-import org.sonar.api.batch.SensorContext;
+import org.mockito.ArgumentMatcher;
+import org.mockito.exceptions.verification.junit.ArgumentsAreDifferent;
+import org.sonar.api.CoreProperties;
+import org.sonar.api.batch.SonarIndex;
+import org.sonar.api.config.Settings;
 import org.sonar.api.resources.AbstractLanguage;
 import org.sonar.api.resources.Java;
 import org.sonar.api.resources.JavaFile;
 import org.sonar.api.resources.Languages;
 import org.sonar.api.resources.Project;
+import org.sonar.api.resources.Qualifiers;
+import org.sonar.api.resources.Resource;
 import org.sonar.api.scan.filesystem.FileQuery;
 import org.sonar.api.scan.filesystem.internal.InputFile;
 import org.sonar.api.scan.filesystem.internal.InputFileBuilder;
@@ -35,8 +46,11 @@ import org.sonar.batch.scan.filesystem.DefaultModuleFileSystem;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.charset.Charset;
 import java.util.Arrays;
 
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
@@ -45,56 +59,176 @@ public class FileIndexerTest {
 
   @Rule
   public TemporaryFolder temp = new TemporaryFolder();
+  private File baseDir;
+  private DefaultModuleFileSystem fs;
+  private SonarIndex sonarIndex;
+  private AbstractLanguage cobolLanguage;
+  private Project project;
+  private Settings settings;
+
+  private String aClaess;
+  private String explicacao;
+
+  @Before
+  public void prepare() throws IOException {
+    baseDir = temp.newFolder();
+    fs = mock(DefaultModuleFileSystem.class);
+    sonarIndex = mock(SonarIndex.class);
+    project = mock(Project.class);
+    settings = new Settings();
+    cobolLanguage = new AbstractLanguage("cobol") {
+      @Override
+      public String[] getFileSuffixes() {
+        return new String[] {"cbl"};
+      }
+    };
+
+    aClaess = new String(new byte[] {65, 67, 108, 97, -61, -88, 115, 115, 40, 41}, CharEncoding.UTF_8);
+    explicacao = new String(new byte[] {101, 120, 112, 108, 105, 99, 97, -61, -89, -61, -93, 111, 40, 41}, CharEncoding.UTF_8);
+  }
 
   @Test
-  public void should_index_java_files() throws IOException {
-    File baseDir = temp.newFolder();
-    DefaultModuleFileSystem fs = mock(DefaultModuleFileSystem.class);
+  public void should_index_java_files() {
     File javaFile1 = new File(baseDir, "src/main/java/foo/bar/Foo.java");
     File javaFile2 = new File(baseDir, "src/main/java2/foo/bar/Foo.java");
     when(fs.inputFiles(FileQuery.onSource().onLanguage(Java.KEY))).thenReturn((Iterable) Arrays.asList(
-      new InputFileBuilder(javaFile1, "src/main/java/foo/bar/Foo.java").attribute(InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "foo/bar/Foo.java").build(),
-      new InputFileBuilder(javaFile2, "src/main/java2/foo/bar/Foo.java").attribute(InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "foo/bar/Foo.java").build()));
+      new InputFileBuilder(javaFile1, Charsets.UTF_8, "src/main/java/foo/bar/Foo.java").attribute(InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "foo/bar/Foo.java").build(),
+      new InputFileBuilder(javaFile2, Charsets.UTF_8, "src/main/java2/foo/bar/Foo.java").attribute(InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "foo/bar/Foo.java").build()));
     File javaTestFile1 = new File(baseDir, "src/test/java/foo/bar/FooTest.java");
-    when(fs.inputFiles(FileQuery.onTest().onLanguage(Java.KEY))).thenReturn((Iterable) Arrays.asList(
-      new InputFileBuilder(javaTestFile1, "src/test/java/foo/bar/FooTest.java").attribute(InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "foo/bar/FooTest.java").build()));
-    Project project = mock(Project.class);
+    when(fs.inputFiles(FileQuery.onTest().onLanguage(Java.KEY))).thenReturn(
+      (Iterable) Arrays.asList(
+        new InputFileBuilder(javaTestFile1, Charsets.UTF_8, "src/test/java/foo/bar/FooTest.java").attribute(InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "foo/bar/FooTest.java")
+          .build()));
     when(project.getLanguageKey()).thenReturn(Java.KEY);
-    FileIndexer indexer = new FileIndexer(project, fs, new Languages(Java.INSTANCE));
-    SensorContext sensorContext = mock(SensorContext.class);
-    indexer.execute(sensorContext);
+    FileIndexer indexer = new FileIndexer(project, fs, new Languages(Java.INSTANCE), sonarIndex, settings);
+    indexer.execute();
 
-    verify(sensorContext).index(new JavaFile("foo.bar.Foo", false).setPath("/src/main/java/foo/bar/Foo.java"));
-    verify(sensorContext).index(new JavaFile("foo.bar.Foo", false).setPath("/src/main/java2/foo/bar/Foo.java"));
-    verify(sensorContext).index(new JavaFile("foo.bar.FooTest", true).setPath("/src/test/java/foo/bar/FooTest.java"));
+    verify(sonarIndex).index(new JavaFile("foo.bar.Foo", false).setPath("/src/main/java/foo/bar/Foo.java"));
+    verify(sonarIndex).index(new JavaFile("foo.bar.Foo", false).setPath("/src/main/java2/foo/bar/Foo.java"));
+    verify(sonarIndex).index(argThat(new ArgumentMatcher<JavaFile>() {
+      @Override
+      public boolean matches(Object arg0) {
+        JavaFile javaFile = (JavaFile) arg0;
+        return javaFile.getKey().equals("foo.bar.FooTest") && javaFile.getPath().equals("/src/test/java/foo/bar/FooTest.java")
+          && javaFile.getQualifier().equals(Qualifiers.UNIT_TEST_FILE);
+      }
+    }));
   }
 
   @Test
   public void should_index_cobol_files() throws IOException {
-    File baseDir = temp.newFolder();
-    DefaultModuleFileSystem fs = mock(DefaultModuleFileSystem.class);
     File cobolFile1 = new File(baseDir, "src/foo/bar/Foo.cbl");
     File cobolFile2 = new File(baseDir, "src2/foo/bar/Foo.cbl");
     when(fs.inputFiles(FileQuery.onSource().onLanguage("cobol"))).thenReturn((Iterable) Arrays.asList(
-      new InputFileBuilder(cobolFile1, "src/foo/bar/Foo.cbl").attribute(InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "foo/bar/Foo.cbl").build(),
-      new InputFileBuilder(cobolFile2, "src2/foo/bar/Foo.cbl").attribute(InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "foo/bar/Foo.cbl").build()));
+      new InputFileBuilder(cobolFile1, Charsets.UTF_8, "src/foo/bar/Foo.cbl").attribute(InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "foo/bar/Foo.cbl").build(),
+      new InputFileBuilder(cobolFile2, Charsets.UTF_8, "src2/foo/bar/Foo.cbl").attribute(InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "foo/bar/Foo.cbl").build()));
     File cobolTestFile1 = new File(baseDir, "src/test/foo/bar/FooTest.cbl");
     when(fs.inputFiles(FileQuery.onTest().onLanguage("cobol"))).thenReturn((Iterable) Arrays.asList(
-      new InputFileBuilder(cobolTestFile1, "src/test/foo/bar/FooTest.cbl").attribute(InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "foo/bar/FooTest.cbl").build()));
-    Project project = mock(Project.class);
+      new InputFileBuilder(cobolTestFile1, Charsets.UTF_8, "src/test/foo/bar/FooTest.cbl").attribute(InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "foo/bar/FooTest.cbl").build()));
     when(project.getLanguageKey()).thenReturn("cobol");
-    FileIndexer indexer = new FileIndexer(project, fs, new Languages(new AbstractLanguage("cobol") {
+
+    FileIndexer indexer = new FileIndexer(project, fs, new Languages(cobolLanguage), sonarIndex, settings);
+    indexer.execute();
+
+    verify(sonarIndex).index(new org.sonar.api.resources.File("foo/bar/Foo.cbl").setPath("/src/foo/bar/Foo.cbl"));
+    verify(sonarIndex).index(new org.sonar.api.resources.File("foo/bar/Foo.cbl").setPath("/src2/foo/bar/Foo.cbl"));
+    verify(sonarIndex).index(new org.sonar.api.resources.File("foo/bar/FooTest.cbl").setPath("/src/test/foo/bar/FooTest.cbl"));
+  }
+
+  @Test
+  public void shouldImportSource() throws IOException {
+    settings.setProperty(CoreProperties.CORE_IMPORT_SOURCES_PROPERTY, "true");
+
+    File javaFile1 = new File(baseDir, "src/main/java/foo/bar/Foo.java");
+    FileUtils.write(javaFile1, "sample code");
+    when(fs.inputFiles(FileQuery.onSource().onLanguage(Java.KEY))).thenReturn((Iterable) Arrays.asList(
+      new InputFileBuilder(javaFile1, Charsets.UTF_8, "src/main/java/foo/bar/Foo.java").attribute(InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "foo/bar/Foo.java").build()));
+    when(fs.inputFiles(FileQuery.onTest().onLanguage(Java.KEY))).thenReturn(
+      (Iterable) Collections.emptyList());
+    when(project.getLanguageKey()).thenReturn(Java.KEY);
+    FileIndexer indexer = new FileIndexer(project, fs, new Languages(Java.INSTANCE), sonarIndex, settings);
+    indexer.execute();
+
+    Resource sonarFile = new JavaFile("foo.bar.Foo", false).setPath("/src/main/java/foo/bar/Foo.java");
+    verify(sonarIndex).index(sonarFile);
+    verify(sonarIndex).setSource(sonarFile, "sample code");
+  }
+
+  @Test
+  public void should_use_mac_roman_charset_forR_reading_source_files() throws Exception {
+    String encoding = "MacRoman";
+    String testFile = "MacRomanEncoding.java";
+    fileEncodingTest(encoding, testFile);
+  }
+
+  @Test
+  public void should_use_CP1252_charset_for_reading_source_files() throws Exception {
+    String encoding = "CP1252";
+    String testFile = "CP1252Encoding.java";
+    fileEncodingTest(encoding, testFile);
+  }
+
+  @Test(expected = ArgumentsAreDifferent.class)
+  public void should_fail_with_wrong_charset_for_reading_source_files() throws Exception {
+    String encoding = CharEncoding.UTF_8;
+    String testFile = "CP1252Encoding.java";
+    fileEncodingTest(encoding, testFile);
+  }
+
+  @Test
+  public void should_remove_byte_order_mark_character() throws Exception {
+    settings.setProperty(CoreProperties.CORE_IMPORT_SOURCES_PROPERTY, "true");
+
+    File javaFile1 = new File(baseDir, "src/main/java/foo/bar/Foo.java");
+    FileUtils.write(javaFile1, "\uFEFFpublic class Test", Charsets.UTF_8);
+    when(fs.inputFiles(FileQuery.onSource().onLanguage(Java.KEY))).thenReturn((Iterable) Arrays.asList(
+      new InputFileBuilder(javaFile1, Charsets.UTF_8, "src/main/java/foo/bar/Foo.java").attribute(InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "foo/bar/Foo.java").build()));
+    when(fs.inputFiles(FileQuery.onTest().onLanguage(Java.KEY))).thenReturn(
+      (Iterable) Collections.emptyList());
+    when(project.getLanguageKey()).thenReturn(Java.KEY);
+    FileIndexer indexer = new FileIndexer(project, fs, new Languages(Java.INSTANCE), sonarIndex, settings);
+    indexer.execute();
+
+    Resource sonarFile = new JavaFile("foo.bar.Foo", false).setPath("/src/main/java/foo/bar/Foo.java");
+
+    verify(sonarIndex).setSource(eq(sonarFile), argThat(new ArgumentMatcher<String>() {
       @Override
-      public String[] getFileSuffixes() {
-        return new String[] {"cbl"};
+      public boolean matches(Object arg0) {
+        String source = (String) arg0;
+        return !source.contains("\uFEFF");
+      }
+    }));
+  }
+
+  private void fileEncodingTest(String encoding, String testFile) throws Exception {
+    settings.setProperty(CoreProperties.CORE_IMPORT_SOURCES_PROPERTY, "true");
+
+    File javaFile1 = new File(baseDir, "src/main/java/foo/bar/Foo.java");
+    FileUtils.copyFile(getFile(testFile), javaFile1);
+    when(fs.inputFiles(FileQuery.onSource().onLanguage(Java.KEY)))
+      .thenReturn(
+        (Iterable) Arrays.asList(
+          new InputFileBuilder(javaFile1, Charset.forName(encoding), "src/main/java/foo/bar/Foo.java").attribute(InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "foo/bar/Foo.java")
+            .build()));
+    when(fs.inputFiles(FileQuery.onTest().onLanguage(Java.KEY))).thenReturn(
+      (Iterable) Collections.emptyList());
+    when(project.getLanguageKey()).thenReturn(Java.KEY);
+    FileIndexer indexer = new FileIndexer(project, fs, new Languages(Java.INSTANCE), sonarIndex, settings);
+    indexer.execute();
+
+    Resource sonarFile = new JavaFile("foo.bar.Foo", false).setPath("/src/main/java/foo/bar/Foo.java");
+
+    verify(sonarIndex).setSource(eq(sonarFile), argThat(new ArgumentMatcher<String>() {
+      @Override
+      public boolean matches(Object arg0) {
+        String source = (String) arg0;
+        return source.contains(aClaess) && source.contains(explicacao);
       }
     }));
-    SensorContext sensorContext = mock(SensorContext.class);
-    indexer.execute(sensorContext);
+  }
 
-    verify(sensorContext).index(new org.sonar.api.resources.File("foo/bar/Foo.cbl").setPath("/src/foo/bar/Foo.cbl"));
-    verify(sensorContext).index(new org.sonar.api.resources.File("foo/bar/Foo.cbl").setPath("/src2/foo/bar/Foo.cbl"));
-    verify(sensorContext).index(new org.sonar.api.resources.File("foo/bar/FooTest.cbl").setPath("/src/test/foo/bar/FooTest.cbl"));
+  private File getFile(String testFile) {
+    return new File("test-resources/org/sonar/batch/phases/FileIndexerTest/encoding/" + testFile);
   }
 
 }
diff --git a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/AttributeFilterTest.java b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/AttributeFilterTest.java
index f168c1acd6d..c5abdd3f9fb 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/AttributeFilterTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/AttributeFilterTest.java
@@ -19,6 +19,7 @@
  */
 package org.sonar.batch.scan.filesystem;
 
+import com.google.common.base.Charsets;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import org.junit.Rule;
@@ -39,9 +40,9 @@ public class AttributeFilterTest {
 
     assertThat(filter.key()).isEqualTo("foo");
     assertThat(filter.values()).containsOnly("one", "two");
-    assertThat(filter.accept(DefaultInputFile.create(temp.newFile(), "Why.java", ImmutableMap.of("foo", "two")))).isTrue();
-    assertThat(filter.accept(DefaultInputFile.create(temp.newFile(), "Where.java", ImmutableMap.of("foo", "three")))).isFalse();
-    assertThat(filter.accept(DefaultInputFile.create(temp.newFile(), "What.java", ImmutableMap.of("bar", "one")))).isFalse();
+    assertThat(filter.accept(DefaultInputFile.create(temp.newFile(), Charsets.UTF_8, "Why.java", ImmutableMap.of("foo", "two")))).isTrue();
+    assertThat(filter.accept(DefaultInputFile.create(temp.newFile(), Charsets.UTF_8, "Where.java", ImmutableMap.of("foo", "three")))).isFalse();
+    assertThat(filter.accept(DefaultInputFile.create(temp.newFile(), Charsets.UTF_8, "What.java", ImmutableMap.of("bar", "one")))).isFalse();
 
   }
 }
diff --git a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystemTest.java b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystemTest.java
index fae404531c1..6c0dbbe079a 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystemTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystemTest.java
@@ -19,6 +19,7 @@
  */
 package org.sonar.batch.scan.filesystem;
 
+import com.google.common.base.Charsets;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import org.junit.Before;
@@ -160,8 +161,8 @@ public class DefaultModuleFileSystemTest {
     DefaultModuleFileSystem fs = new DefaultModuleFileSystem(new Project("foo"), settings, fileIndex, initializer, mode);
 
     File mainFile = temp.newFile();
-    InputFile mainInput = DefaultInputFile.create(mainFile, "Main.java", ImmutableMap.of(InputFile.ATTRIBUTE_TYPE, InputFile.TYPE_SOURCE));
-    InputFile testInput = DefaultInputFile.create(temp.newFile(), "Test.java", ImmutableMap.of(InputFile.ATTRIBUTE_TYPE, InputFile.TYPE_TEST));
+    InputFile mainInput = DefaultInputFile.create(mainFile, Charsets.UTF_8, "Main.java", ImmutableMap.of(InputFile.ATTRIBUTE_TYPE, InputFile.TYPE_SOURCE));
+    InputFile testInput = DefaultInputFile.create(temp.newFile(), Charsets.UTF_8, "Test.java", ImmutableMap.of(InputFile.ATTRIBUTE_TYPE, InputFile.TYPE_TEST));
 
     when(fileIndex.inputFiles("foo")).thenReturn(Lists.newArrayList(mainInput, testInput));
 
diff --git a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/DeprecatedFileFiltersTest.java b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/DeprecatedFileFiltersTest.java
index 94f1e31ebad..58ba069883d 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/DeprecatedFileFiltersTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/DeprecatedFileFiltersTest.java
@@ -19,6 +19,7 @@
  */
 package org.sonar.batch.scan.filesystem;
 
+import com.google.common.base.Charsets;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 import org.apache.commons.io.FilenameUtils;
@@ -28,15 +29,17 @@ import org.junit.rules.TemporaryFolder;
 import org.mockito.ArgumentCaptor;
 import org.sonar.api.scan.filesystem.FileSystemFilter;
 import org.sonar.api.scan.filesystem.FileType;
-import org.sonar.api.scan.filesystem.internal.InputFile;
 import org.sonar.api.scan.filesystem.internal.DefaultInputFile;
+import org.sonar.api.scan.filesystem.internal.InputFile;
 
 import java.io.File;
 
 import static org.fest.assertions.Assertions.assertThat;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 public class DeprecatedFileFiltersTest {
 
@@ -49,22 +52,22 @@ public class DeprecatedFileFiltersTest {
   public void no_filters() throws Exception {
     DeprecatedFileFilters filters = new DeprecatedFileFilters();
 
-    InputFile inputFile = DefaultInputFile.create(temp.newFile(), "src/main/java/Foo.java", Maps.<String, String>newHashMap());
+    InputFile inputFile = DefaultInputFile.create(temp.newFile(), Charsets.UTF_8, "src/main/java/Foo.java", Maps.<String, String>newHashMap());
     assertThat(filters.accept(inputFile)).isTrue();
   }
 
   @Test
   public void at_least_one_filter() throws Exception {
-    DeprecatedFileFilters filters = new DeprecatedFileFilters(new FileSystemFilter[]{filter});
+    DeprecatedFileFilters filters = new DeprecatedFileFilters(new FileSystemFilter[] {filter});
 
     File basedir = temp.newFolder();
     File file = temp.newFile();
-    InputFile inputFile = DefaultInputFile.create(file, "src/main/java/Foo.java", ImmutableMap.of(
+    InputFile inputFile = DefaultInputFile.create(file, Charsets.UTF_8, "src/main/java/Foo.java", ImmutableMap.of(
       InputFile.ATTRIBUTE_SOURCEDIR_PATH, new File(basedir, "src/main/java").getAbsolutePath(),
       InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "Foo.java",
       InputFile.ATTRIBUTE_TYPE, InputFile.TYPE_TEST
 
-    ));
+      ));
     when(filter.accept(eq(file), any(DeprecatedFileFilters.DeprecatedContext.class))).thenReturn(false);
 
     assertThat(filters.accept(inputFile)).isFalse();
diff --git a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/ExclusionFilterTest.java b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/ExclusionFilterTest.java
index 39df30086b2..66c2990ec24 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/ExclusionFilterTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/ExclusionFilterTest.java
@@ -19,12 +19,13 @@
  */
 package org.sonar.batch.scan.filesystem;
 
+import com.google.common.base.Charsets;
 import com.google.common.collect.ImmutableMap;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
-import org.sonar.api.scan.filesystem.internal.InputFile;
 import org.sonar.api.scan.filesystem.internal.DefaultInputFile;
+import org.sonar.api.scan.filesystem.internal.InputFile;
 
 import java.io.File;
 
@@ -41,17 +42,17 @@ public class ExclusionFilterTest {
     ExclusionFilter absoluteFilter = new ExclusionFilter("file:**/src/main/**Foo.java");
 
     File file = new File(temp.newFolder(), "src/main/java/org/MyFoo.java");
-    InputFile inputFile = DefaultInputFile.create(file, "src/main/java/org/MyFoo.java", ImmutableMap.of(
+    InputFile inputFile = DefaultInputFile.create(file, Charsets.UTF_8, "src/main/java/org/MyFoo.java", ImmutableMap.of(
       InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "org/MyFoo.java"
-    ));
+      ));
 
     assertThat(sourceRelativeFilter.accept(inputFile)).isFalse();
     assertThat(absoluteFilter.accept(inputFile)).isFalse();
 
     file = new File(temp.newFolder(), "src/main/java/org/Other.java");
-    inputFile = DefaultInputFile.create(file, "src/main/java/org/Other.java", ImmutableMap.of(
+    inputFile = DefaultInputFile.create(file, Charsets.UTF_8, "src/main/java/org/Other.java", ImmutableMap.of(
       InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "org/Other.java"
-    ));
+      ));
     assertThat(sourceRelativeFilter.accept(inputFile)).isTrue();
     assertThat(absoluteFilter.accept(inputFile)).isTrue();
   }
diff --git a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/ExclusionFiltersTest.java b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/ExclusionFiltersTest.java
index a80d153643a..8c913d24b79 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/ExclusionFiltersTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/ExclusionFiltersTest.java
@@ -19,6 +19,7 @@
  */
 package org.sonar.batch.scan.filesystem;
 
+import com.google.common.base.Charsets;
 import com.google.common.collect.ImmutableMap;
 import org.junit.Rule;
 import org.junit.Test;
@@ -29,8 +30,8 @@ import org.sonar.api.resources.File;
 import org.sonar.api.resources.JavaFile;
 import org.sonar.api.resources.Project;
 import org.sonar.api.scan.filesystem.FileExclusions;
-import org.sonar.api.scan.filesystem.internal.InputFile;
 import org.sonar.api.scan.filesystem.internal.DefaultInputFile;
+import org.sonar.api.scan.filesystem.internal.InputFile;
 
 import java.io.IOException;
 
@@ -48,24 +49,24 @@ public class ExclusionFiltersTest {
     ExclusionFilters filter = new ExclusionFilters(new FileExclusions(settings));
 
     java.io.File file = temp.newFile();
-    InputFile inputFile = DefaultInputFile.create(file, "src/main/java/com/mycompany/Foo.java", ImmutableMap.of(
+    InputFile inputFile = DefaultInputFile.create(file, Charsets.UTF_8, "src/main/java/com/mycompany/Foo.java", ImmutableMap.of(
       InputFile.ATTRIBUTE_TYPE, InputFile.TYPE_SOURCE,
       InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "com/mycompany/Foo.java"
-    ));
+      ));
 
     assertThat(filter.accept(inputFile)).isFalse();
 
-    inputFile = DefaultInputFile.create(file, "src/main/java/com/mycompany/FooDao.java", ImmutableMap.of(
+    inputFile = DefaultInputFile.create(file, Charsets.UTF_8, "src/main/java/com/mycompany/FooDao.java", ImmutableMap.of(
       InputFile.ATTRIBUTE_TYPE, InputFile.TYPE_SOURCE,
       InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "com/mycompany/FooDao.java"
-    ));
+      ));
     assertThat(filter.accept(inputFile)).isTrue();
 
     // source inclusions do not apply to tests
-    inputFile = DefaultInputFile.create(file, "src/main/java/com/mycompany/Foo.java", ImmutableMap.of(
+    inputFile = DefaultInputFile.create(file, Charsets.UTF_8, "src/main/java/com/mycompany/Foo.java", ImmutableMap.of(
       InputFile.ATTRIBUTE_TYPE, InputFile.TYPE_TEST,
       InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "com/mycompany/Foo.java"
-    ));
+      ));
     assertThat(filter.accept(inputFile)).isTrue();
   }
 
@@ -76,17 +77,17 @@ public class ExclusionFiltersTest {
     ExclusionFilters filter = new ExclusionFilters(new FileExclusions(settings));
 
     java.io.File file = temp.newFile();
-    InputFile inputFile = DefaultInputFile.create(file, "src/main/java/com/mycompany/Foo.java", ImmutableMap.of(
+    InputFile inputFile = DefaultInputFile.create(file, Charsets.UTF_8, "src/main/java/com/mycompany/Foo.java", ImmutableMap.of(
       InputFile.ATTRIBUTE_TYPE, InputFile.TYPE_SOURCE,
       InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "com/mycompany/Foo.java"
-    ));
+      ));
 
     assertThat(filter.accept(inputFile)).isFalse();
 
-    inputFile = DefaultInputFile.create(file, "src/main/java/com/mycompany/FooDto.java", ImmutableMap.of(
+    inputFile = DefaultInputFile.create(file, Charsets.UTF_8, "src/main/java/com/mycompany/FooDto.java", ImmutableMap.of(
       InputFile.ATTRIBUTE_TYPE, InputFile.TYPE_SOURCE,
       InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "com/mycompany/FooDto.java"
-    ));
+      ));
     assertThat(filter.accept(inputFile)).isTrue();
   }
 
@@ -97,23 +98,23 @@ public class ExclusionFiltersTest {
     ExclusionFilters filter = new ExclusionFilters(new FileExclusions(settings));
 
     java.io.File file = temp.newFile();
-    InputFile inputFile = DefaultInputFile.create(file, "src/main/java/com/mycompany/FooDao.java", ImmutableMap.of(
+    InputFile inputFile = DefaultInputFile.create(file, Charsets.UTF_8, "src/main/java/com/mycompany/FooDao.java", ImmutableMap.of(
       InputFile.ATTRIBUTE_TYPE, InputFile.TYPE_SOURCE,
       InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "com/mycompany/FooDao.java"
-    ));
+      ));
     assertThat(filter.accept(inputFile)).isFalse();
 
-    inputFile = DefaultInputFile.create(file, "src/main/java/com/mycompany/Foo.java", ImmutableMap.of(
+    inputFile = DefaultInputFile.create(file, Charsets.UTF_8, "src/main/java/com/mycompany/Foo.java", ImmutableMap.of(
       InputFile.ATTRIBUTE_TYPE, InputFile.TYPE_SOURCE,
       InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "com/mycompany/Foo.java"
-    ));
+      ));
     assertThat(filter.accept(inputFile)).isTrue();
 
     // source exclusions do not apply to tests
-    inputFile = DefaultInputFile.create(file, "src/main/java/com/mycompany/FooDao.java", ImmutableMap.of(
+    inputFile = DefaultInputFile.create(file, Charsets.UTF_8, "src/main/java/com/mycompany/FooDao.java", ImmutableMap.of(
       InputFile.ATTRIBUTE_TYPE, InputFile.TYPE_TEST,
       InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "com/mycompany/FooDao.java"
-    ));
+      ));
     assertThat(filter.accept(inputFile)).isTrue();
   }
 
@@ -126,14 +127,14 @@ public class ExclusionFiltersTest {
     settings.setProperty(CoreProperties.PROJECT_EXCLUSIONS_PROPERTY, "file:" + excludedFile.getCanonicalPath());
     ExclusionFilters filter = new ExclusionFilters(new FileExclusions(settings));
 
-    InputFile includedInput = DefaultInputFile.create(includedFile, "src/main/java/org/bar/Foo.java", ImmutableMap.of(
+    InputFile includedInput = DefaultInputFile.create(includedFile, Charsets.UTF_8, "src/main/java/org/bar/Foo.java", ImmutableMap.of(
       InputFile.ATTRIBUTE_TYPE, InputFile.TYPE_SOURCE
-    ));
+      ));
     assertThat(filter.accept(includedInput)).isTrue();
 
-    InputFile excludedInput = DefaultInputFile.create(excludedFile, "src/main/java/org/bar/Bar.java", ImmutableMap.of(
+    InputFile excludedInput = DefaultInputFile.create(excludedFile, Charsets.UTF_8, "src/main/java/org/bar/Bar.java", ImmutableMap.of(
       InputFile.ATTRIBUTE_TYPE, InputFile.TYPE_SOURCE
-    ));
+      ));
     assertThat(filter.accept(excludedInput)).isFalse();
   }
 
@@ -236,5 +237,4 @@ public class ExclusionFiltersTest {
     assertThat(filter.sourceExclusions()[0].toString()).isEqualTo("**/*Dao.java");
   }
 
-
 }
diff --git a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InclusionFilterTest.java b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InclusionFilterTest.java
index 2be6fbb6b24..cd76e714c66 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InclusionFilterTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InclusionFilterTest.java
@@ -19,12 +19,13 @@
  */
 package org.sonar.batch.scan.filesystem;
 
+import com.google.common.base.Charsets;
 import com.google.common.collect.ImmutableMap;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
-import org.sonar.api.scan.filesystem.internal.InputFile;
 import org.sonar.api.scan.filesystem.internal.DefaultInputFile;
+import org.sonar.api.scan.filesystem.internal.InputFile;
 
 import java.io.File;
 
@@ -40,17 +41,17 @@ public class InclusionFilterTest {
     InclusionFilter absoluteFilter = new InclusionFilter("file:**/src/main/**Foo.java");
 
     File file = new File(temp.newFolder(), "src/main/java/org/MyFoo.java");
-    InputFile inputFile = DefaultInputFile.create(file, "src/main/java/org/MyFoo.java", ImmutableMap.of(
+    InputFile inputFile = DefaultInputFile.create(file, Charsets.UTF_8, "src/main/java/org/MyFoo.java", ImmutableMap.of(
       InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "org/MyFoo.java"
-    ));
+      ));
 
     assertThat(sourceRelativeFilter.accept(inputFile)).isTrue();
     assertThat(absoluteFilter.accept(inputFile)).isTrue();
 
     file = new File(temp.newFolder(), "src/main/java/org/Other.java");
-    inputFile = DefaultInputFile.create(file, "src/main/java/org/Other.java", ImmutableMap.of(
+    inputFile = DefaultInputFile.create(file, Charsets.UTF_8, "src/main/java/org/Other.java", ImmutableMap.of(
       InputFile.ATTRIBUTE_SOURCE_RELATIVE_PATH, "org/Other.java"
-    ));
+      ));
     assertThat(sourceRelativeFilter.accept(inputFile)).isFalse();
     assertThat(absoluteFilter.accept(inputFile)).isFalse();
   }
diff --git a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InputFileCacheTest.java b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InputFileCacheTest.java
index 5347d808ac6..50635adc091 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InputFileCacheTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/InputFileCacheTest.java
@@ -19,14 +19,15 @@
  */
 package org.sonar.batch.scan.filesystem;
 
+import com.google.common.base.Charsets;
 import com.google.common.collect.Maps;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
-import org.sonar.api.scan.filesystem.internal.InputFile;
 import org.sonar.api.scan.filesystem.internal.DefaultInputFile;
+import org.sonar.api.scan.filesystem.internal.InputFile;
 import org.sonar.batch.index.Caches;
 import org.sonar.batch.index.CachesTest;
 
@@ -53,8 +54,8 @@ public class InputFileCacheTest {
   @Test
   public void should_add_input_file() throws Exception {
     InputFileCache cache = new InputFileCache(caches);
-    cache.put("struts", DefaultInputFile.create(temp.newFile(), "src/main/java/Foo.java", Maps.<String, String>newHashMap()));
-    cache.put("struts-core", DefaultInputFile.create(temp.newFile(), "src/main/java/Foo.java", Maps.<String, String>newHashMap()));
+    cache.put("struts", DefaultInputFile.create(temp.newFile(), Charsets.UTF_8, "src/main/java/Foo.java", Maps.<String, String>newHashMap()));
+    cache.put("struts-core", DefaultInputFile.create(temp.newFile(), Charsets.UTF_8, "src/main/java/Foo.java", Maps.<String, String>newHashMap()));
 
     assertThat(cache.byModule("struts")).hasSize(1);
     assertThat(cache.byModule("struts-core")).hasSize(1);
diff --git a/sonar-batch/test-resources/org/sonar/batch/phases/FileIndexerTest/encoding/CP1252Encoding.java b/sonar-batch/test-resources/org/sonar/batch/phases/FileIndexerTest/encoding/CP1252Encoding.java
new file mode 100644
index 00000000000..5f80ef59765
--- /dev/null
+++ b/sonar-batch/test-resources/org/sonar/batch/phases/FileIndexerTest/encoding/CP1252Encoding.java
@@ -0,0 +1,13 @@
+public class Car {
+
+	public AClaèss() {
+	}
+
+	public int explicação() {
+	  return 1;
+	}
+
+	public String getS() {
+		return "";
+	}
+}
diff --git a/sonar-batch/test-resources/org/sonar/batch/phases/FileIndexerTest/encoding/MacRomanEncoding.java b/sonar-batch/test-resources/org/sonar/batch/phases/FileIndexerTest/encoding/MacRomanEncoding.java
new file mode 100644
index 00000000000..30e52000eaf
--- /dev/null
+++ b/sonar-batch/test-resources/org/sonar/batch/phases/FileIndexerTest/encoding/MacRomanEncoding.java
@@ -0,0 +1,13 @@
+public class Car {
+
+  public ACla�ss() {
+  }
+
+  public int explica�‹o() {
+    return 1;
+  }
+
+  public String getS() {
+    return "";
+  }
+}