SONAR-9204 File with BOM are not correctly parsed by the CPD tokenizer

2 files changed, 17 insertions, 17 deletions

diff --git a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/XooTokenizer.java b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/XooTokenizer.java
index d57467c9859..0aa6bffb27c 100644
--- a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/XooTokenizer.java
+++ b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/XooTokenizer.java
@@ -48,16 +48,12 @@ public class XooTokenizer implements Tokenizer {
     String fileName = source.getFileName();
     LOG.info("Using deprecated tokenizer extension point to tokenize {}", fileName);
     int lineIdx = 1;
-    try {
-      for (String line : FileUtils.readLines(new File(fileName), fs.encoding())) {
-        for (String token : Splitter.on(" ").split(line)) {
-          TokenEntry cpdToken = new TokenEntry(token, fileName, lineIdx);
-          cpdTokens.add(cpdToken);
-        }
-        lineIdx++;
+    for (String line : source.getCode()) {
+      for (String token : Splitter.on(" ").split(line)) {
+        TokenEntry cpdToken = new TokenEntry(token, fileName, lineIdx);
+        cpdTokens.add(cpdToken);
       }
-    } catch (IOException e) {
-      throw new IllegalStateException("Unable to tokenize", e);
+      lineIdx++;
     }
     cpdTokens.add(TokenEntry.getEOF());
   }
diff --git a/plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/lang/XooTokenizerTest.java b/plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/lang/XooTokenizerTest.java
index c6a5cbf9859..967f87dafdb 100644
--- a/plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/lang/XooTokenizerTest.java
+++ b/plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/lang/XooTokenizerTest.java
@@ -19,6 +19,10 @@
  */
 package org.sonar.xoo.lang;
 
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
 import net.sourceforge.pmd.cpd.SourceCode;
 import net.sourceforge.pmd.cpd.TokenEntry;
 import net.sourceforge.pmd.cpd.Tokens;
@@ -31,11 +35,9 @@ import org.sonar.api.batch.fs.internal.DefaultFileSystem;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.batch.fs.internal.TestInputFileBuilder;
 import org.sonar.api.batch.sensor.SensorContext;
-import org.sonar.api.config.Settings;
-
-import java.io.File;
-import java.io.IOException;
 import org.sonar.api.config.MapSettings;
+import org.sonar.api.config.Settings;
+import org.sonar.duplications.cpd.FileCodeLoaderWithoutCache;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.mock;
@@ -63,18 +65,20 @@ public class XooTokenizerTest {
   @Test
   public void testExecution() throws IOException {
     File source = new File(baseDir, "src/foo.xoo");
-    FileUtils.write(source, "token1 token2 token3\ntoken4");
+    FileUtils.write(source, "token1 token2 token3\ntoken4", StandardCharsets.UTF_8);
     DefaultInputFile inputFile = new TestInputFileBuilder("foo", "src/foo.xoo")
       .setLanguage("xoo")
       .setModuleBaseDir(baseDir.toPath())
+      .setCharset(StandardCharsets.UTF_8)
       .build();
     fileSystem.add(inputFile);
 
     XooTokenizer tokenizer = new XooTokenizer(fileSystem);
-    SourceCode sourceCode = mock(SourceCode.class);
-    when(sourceCode.getFileName()).thenReturn(inputFile.absolutePath());
     Tokens cpdTokens = new Tokens();
-    tokenizer.tokenize(sourceCode, cpdTokens);
+    try (InputStreamReader reader = new InputStreamReader(inputFile.inputStream(), inputFile.charset())) {
+      SourceCode sourceCode = new SourceCode(new FileCodeLoaderWithoutCache(inputFile.absolutePath(), reader));
+      tokenizer.tokenize(sourceCode, cpdTokens);
+    }
 
     // 4 tokens + EOF
     assertThat(cpdTokens.getTokens()).hasSize(5);