SONAR-1091 Use correct encoding during detection of duplications

3 files changed, 41 insertions, 7 deletions

diff --git a/sonar-duplications/src/test/java/org/sonar/duplications/DuplicationsTestUtil.java b/sonar-duplications/src/test/java/org/sonar/duplications/DuplicationsTestUtil.java
index c1a37bba1ae..9e1072eb4b0 100644
--- a/sonar-duplications/src/test/java/org/sonar/duplications/DuplicationsTestUtil.java
+++ b/sonar-duplications/src/test/java/org/sonar/duplications/DuplicationsTestUtil.java
@@ -23,7 +23,7 @@ import java.io.File;
 
 public class DuplicationsTestUtil {
 
-  public static final File fileDir = new File("src/test/files/");
+  private static final File fileDir = new File("src/test/files/");
 
   public static File findFile(String relativePathToFile) {
     return new File(fileDir, relativePathToFile);
diff --git a/sonar-duplications/src/test/java/org/sonar/duplications/java/JavaStatementBuilderTest.java b/sonar-duplications/src/test/java/org/sonar/duplications/java/JavaStatementBuilderTest.java
index 2267fa8a125..fa579e21b34 100644
--- a/sonar-duplications/src/test/java/org/sonar/duplications/java/JavaStatementBuilderTest.java
+++ b/sonar-duplications/src/test/java/org/sonar/duplications/java/JavaStatementBuilderTest.java
@@ -24,8 +24,14 @@ import static org.hamcrest.number.OrderingComparisons.greaterThan;
 import static org.junit.Assert.assertThat;
 
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.charset.Charset;
 import java.util.List;
 
+import org.apache.commons.io.IOUtils;
 import org.junit.Test;
 import org.sonar.duplications.DuplicationsTestUtil;
 import org.sonar.duplications.statement.Statement;
@@ -150,11 +156,20 @@ public class JavaStatementBuilderTest {
 
   @Test
   public void realExamples() {
-    File testFile = DuplicationsTestUtil.findFile("/java/MessageResources.java");
-    assertThat(statementChunker.chunk(tokenChunker.chunk(testFile)).size(), greaterThan(0));
+    assertThat(chunk(DuplicationsTestUtil.findFile("/java/MessageResources.java")).size(), greaterThan(0));
+    assertThat(chunk(DuplicationsTestUtil.findFile("/java/RequestUtils.java")).size(), greaterThan(0));
+  }
 
-    testFile = DuplicationsTestUtil.findFile("/java/RequestUtils.java");
-    assertThat(statementChunker.chunk(tokenChunker.chunk(testFile)).size(), greaterThan(0));
+  private List<Statement> chunk(File file) {
+    Reader reader = null;
+    try {
+      reader = new InputStreamReader(new FileInputStream(file), Charset.forName("UTF-8"));
+      return statementChunker.chunk(tokenChunker.chunk(reader));
+    } catch (FileNotFoundException e) {
+      throw new RuntimeException(e);
+    } finally {
+      IOUtils.closeQuietly(reader);
+    }
   }
 
 }
diff --git a/sonar-duplications/src/test/java/org/sonar/duplications/java/JavaTokenProducerTest.java b/sonar-duplications/src/test/java/org/sonar/duplications/java/JavaTokenProducerTest.java
index 285cdf2c61e..8a2add96005 100644
--- a/sonar-duplications/src/test/java/org/sonar/duplications/java/JavaTokenProducerTest.java
+++ b/sonar-duplications/src/test/java/org/sonar/duplications/java/JavaTokenProducerTest.java
@@ -24,14 +24,21 @@ import static org.hamcrest.number.OrderingComparisons.greaterThan;
 import static org.junit.Assert.assertThat;
 
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.charset.Charset;
 import java.util.Arrays;
 import java.util.List;
 
+import org.apache.commons.io.IOUtils;
 import org.hamcrest.Matcher;
 import org.junit.Test;
 import org.sonar.duplications.DuplicationsTestUtil;
 import org.sonar.duplications.token.Token;
 import org.sonar.duplications.token.TokenChunker;
+import org.sonar.duplications.token.TokenQueue;
 
 import com.google.common.collect.Lists;
 
@@ -265,10 +272,22 @@ public class JavaTokenProducerTest {
   @Test
   public void realExamples() {
     File testFile = DuplicationsTestUtil.findFile("/java/MessageResources.java");
-    assertThat(chunker.chunk(testFile).size(), greaterThan(0));
+    assertThat(chunk(testFile).size(), greaterThan(0));
 
     testFile = DuplicationsTestUtil.findFile("/java/RequestUtils.java");
-    assertThat(chunker.chunk(testFile).size(), greaterThan(0));
+    assertThat(chunk(testFile).size(), greaterThan(0));
+  }
+
+  private TokenQueue chunk(File file) {
+    Reader reader = null;
+    try {
+      reader = new InputStreamReader(new FileInputStream(file), Charset.forName("UTF-8"));
+      return chunker.chunk(reader);
+    } catch (FileNotFoundException e) {
+      throw new RuntimeException(e);
+    } finally {
+      IOUtils.closeQuietly(reader);
+    }
   }
 
   private static Matcher<List<Token>> isNumericLiteral() {