SONAR-9204 File with BOM are not correctly parsed by the CPD tokenizer

5 files changed, 28 insertions, 19 deletions

diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/cpd/deprecated/DefaultCpdBlockIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cpd/deprecated/DefaultCpdBlockIndexer.java
index adeaf2c20d1..587a88a7f16 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/cpd/deprecated/DefaultCpdBlockIndexer.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cpd/deprecated/DefaultCpdBlockIndexer.java
@@ -21,6 +21,8 @@ package org.sonar.scanner.cpd.deprecated;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.io.InputStreamReader;
 import java.util.List;
 import org.sonar.api.CoreProperties;
 import org.sonar.api.batch.CpdMapping;
@@ -80,12 +82,17 @@ public class DefaultCpdBlockIndexer extends CpdBlockIndexer {
   }
 
   private void populateIndex(String languageKey, List<InputFile> sourceFiles, CpdMapping mapping) {
-    TokenizerBridge bridge = new TokenizerBridge(mapping.getTokenizer(), fs.encoding().name(), getBlockSize(languageKey));
+    TokenizerBridge bridge = new TokenizerBridge(mapping.getTokenizer(), getBlockSize(languageKey));
     for (InputFile inputFile : sourceFiles) {
       if (!index.isIndexed(inputFile)) {
         LOG.debug("Populating index from {}", inputFile.absolutePath());
         String resourceEffectiveKey = ((DefaultInputFile) inputFile).key();
-        List<Block> blocks = bridge.chunk(resourceEffectiveKey, inputFile.file());
+        List<Block> blocks;
+        try (InputStreamReader isr = new InputStreamReader(inputFile.inputStream(), inputFile.charset())) {
+          blocks = bridge.chunk(resourceEffectiveKey, inputFile.absolutePath(), isr);
+        } catch (IOException e) {
+          throw new IllegalStateException("Unable to read content of file " + inputFile.absolutePath(), e);
+        }
         index.insert(inputFile, blocks);
       }
     }
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/cpd/deprecated/DeprecatedCpdBlockIndexerSensor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cpd/deprecated/DeprecatedCpdBlockIndexerSensor.java
index b8721d36167..862ee124d9e 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/cpd/deprecated/DeprecatedCpdBlockIndexerSensor.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cpd/deprecated/DeprecatedCpdBlockIndexerSensor.java
@@ -72,7 +72,7 @@ public class DeprecatedCpdBlockIndexerSensor implements Sensor {
         LOG.debug("Detection of duplicated code is not supported for {}", language);
         continue;
       }
-      LOG.info("{} is used for {}", blockIndexer, language);
+      LOG.debug("{} is used for {}", blockIndexer.getClass().getName(), language);
       blockIndexer.index(language);
     }
   }
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/cpd/deprecated/JavaCpdBlockIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cpd/deprecated/JavaCpdBlockIndexer.java
index dc1deca096d..fc0e105d2c1 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/cpd/deprecated/JavaCpdBlockIndexer.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cpd/deprecated/JavaCpdBlockIndexer.java
@@ -20,7 +20,6 @@
 package org.sonar.scanner.cpd.deprecated;
 
 import com.google.common.collect.Lists;
-import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
@@ -91,8 +90,8 @@ public class JavaCpdBlockIndexer extends CpdBlockIndexer {
 
       List<Statement> statements;
 
-      try (InputStream is = new FileInputStream(inputFile.file());
-        Reader reader = new InputStreamReader(is, fs.encoding())) {
+      try (InputStream is = inputFile.inputStream();
+        Reader reader = new InputStreamReader(is, inputFile.charset())) {
         statements = statementChunker.chunk(tokenChunker.chunk(reader));
       } catch (FileNotFoundException e) {
         throw new IllegalStateException("Cannot find file " + inputFile.file(), e);
@@ -100,7 +99,12 @@ public class JavaCpdBlockIndexer extends CpdBlockIndexer {
         throw new IllegalStateException("Exception handling file: " + inputFile.file(), e);
       }
 
-      List<Block> blocks = blockChunker.chunk(resourceEffectiveKey, statements);
+      List<Block> blocks;
+      try {
+        blocks = blockChunker.chunk(resourceEffectiveKey, statements);
+      } catch (Exception e) {
+        throw new IllegalStateException("Cannot process file " + inputFile.file(), e);
+      }
       index.insert(inputFile, blocks);
     }
   }
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/SourcePublisher.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/SourcePublisher.java
index a161fa33835..131a89c4caa 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/SourcePublisher.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/SourcePublisher.java
@@ -19,20 +19,17 @@
  */
 package org.sonar.scanner.report;
 
-import org.apache.commons.io.ByteOrderMark;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.io.input.BOMInputStream;
-import org.sonar.api.batch.fs.internal.DefaultInputFile;
-import org.sonar.scanner.protocol.output.ScannerReportWriter;
-import org.sonar.scanner.scan.filesystem.InputComponentStore;
-
 import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
+import org.apache.commons.io.IOUtils;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.scanner.protocol.output.ScannerReportWriter;
+import org.sonar.scanner.scan.filesystem.InputComponentStore;
 
 public class SourcePublisher implements ReportPublisherStep {
 
@@ -48,9 +45,8 @@ public class SourcePublisher implements ReportPublisherStep {
       File iofile = writer.getSourceFile(inputFile.batchId());
 
       try (FileOutputStream output = new FileOutputStream(iofile);
-        BOMInputStream bomIn = new BOMInputStream(new FileInputStream(inputFile.file()),
-          ByteOrderMark.UTF_8, ByteOrderMark.UTF_16LE, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_32LE, ByteOrderMark.UTF_32BE);
-        BufferedReader reader = new BufferedReader(new InputStreamReader(bomIn, inputFile.charset()))) {
+        InputStream in = inputFile.inputStream();
+        BufferedReader reader = new BufferedReader(new InputStreamReader(in, inputFile.charset()))) {
         writeSource(reader, output, inputFile.lines());
       } catch (IOException e) {
         throw new IllegalStateException("Unable to store file source in the report", e);
diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/cpd/deprecated/JavaCpdBlockIndexerTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/cpd/deprecated/JavaCpdBlockIndexerTest.java
index 29c37b61869..325bb08789b 100644
--- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/cpd/deprecated/JavaCpdBlockIndexerTest.java
+++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/cpd/deprecated/JavaCpdBlockIndexerTest.java
@@ -21,6 +21,7 @@ package org.sonar.scanner.cpd.deprecated;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.List;
 import org.apache.commons.io.FileUtils;
 import org.junit.Before;
@@ -36,8 +37,8 @@ import org.sonar.api.batch.fs.FileSystem;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.internal.DefaultFileSystem;
 import org.sonar.api.batch.fs.internal.TestInputFileBuilder;
-import org.sonar.api.config.Settings;
 import org.sonar.api.config.MapSettings;
+import org.sonar.api.config.Settings;
 import org.sonar.duplications.block.Block;
 import org.sonar.scanner.cpd.index.SonarCpdBlockIndex;
 
@@ -71,6 +72,7 @@ public class JavaCpdBlockIndexerTest {
     DefaultFileSystem fs = new DefaultFileSystem(baseDir);
     file = new TestInputFileBuilder("foo", "src/ManyStatements.java")
       .setModuleBaseDir(baseDir.toPath())
+      .setCharset(StandardCharsets.UTF_8)
       .setLanguage(JAVA).build();
     fs.add(file);
     File ioFile = file.file();