SONAR-8250 Non Ascii characters are not handled properly

4 files changed, 24 insertions, 34 deletions

diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scm/DefaultBlameOutput.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scm/DefaultBlameOutput.java
index cc56fadfd19..f6758b03385 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scm/DefaultBlameOutput.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scm/DefaultBlameOutput.java
@@ -20,14 +20,12 @@
 package org.sonar.scanner.scm;
 
 import com.google.common.base.Preconditions;
-import java.text.Normalizer;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
-import java.util.regex.Pattern;
 import javax.annotation.Nullable;
 import org.apache.commons.lang.StringUtils;
 import org.sonar.api.batch.fs.InputFile;
@@ -45,9 +43,6 @@ class DefaultBlameOutput implements BlameOutput {
 
   private static final Logger LOG = Loggers.get(DefaultBlameOutput.class);
 
-  private static final Pattern NON_ASCII_CHARS = Pattern.compile("[^\\x00-\\x7F]");
-  private static final Pattern ACCENT_CODES = Pattern.compile("\\p{InCombiningDiacriticalMarks}+");
-
   private final ScannerReportWriter writer;
   private final Set<InputFile> allFilesToBlame = new HashSet<>();
   private ProgressReport progressReport;
@@ -117,18 +112,7 @@ class DefaultBlameOutput implements BlameOutput {
     if (inputString == null) {
       return "";
     }
-    String lowerCasedString = inputString.toLowerCase();
-    String stringWithoutAccents = removeAccents(lowerCasedString);
-    return removeNonAsciiCharacters(stringWithoutAccents);
-  }
-
-  private static String removeAccents(String inputString) {
-    String unicodeDecomposedString = Normalizer.normalize(inputString, Normalizer.Form.NFD);
-    return ACCENT_CODES.matcher(unicodeDecomposedString).replaceAll("");
-  }
-
-  private static String removeNonAsciiCharacters(String inputString) {
-    return NON_ASCII_CHARS.matcher(inputString).replaceAll("_");
+    return inputString.toLowerCase();
   }
 
   public void finish(boolean success) {
diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/scm/ScmMediumTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/scm/ScmMediumTest.java
index cd369da5e54..06c3e33f6ae 100644
--- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/scm/ScmMediumTest.java
+++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/scm/ScmMediumTest.java
@@ -22,8 +22,12 @@ package org.sonar.scanner.mediumtest.scm;
 import com.google.common.collect.ImmutableMap;
 import java.io.File;
 import java.io.IOException;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.io.FileUtils;
+import org.assertj.core.util.Files;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
@@ -85,7 +89,7 @@ public class ScmMediumTest {
   }
 
   @Test
-  public void testScmMeasure() throws IOException {
+  public void testScmMeasure() throws IOException, URISyntaxException {
     File baseDir = prepareProject();
 
     tester.newTask()
@@ -109,7 +113,7 @@ public class ScmMediumTest {
     assertThat(changesetLine1.getAuthor()).isEmpty();
 
     Changeset changesetLine2 = fileScm.getChangeset(fileScm.getChangesetIndexByLine(1));
-    assertThat(changesetLine2.getAuthor()).isEqualTo("julien");
+    assertThat(changesetLine2.getAuthor()).isEqualTo(getNonAsciiAuthor().toLowerCase());
 
     Changeset changesetLine3 = fileScm.getChangeset(fileScm.getChangesetIndexByLine(2));
     assertThat(changesetLine3.getAuthor()).isEqualTo("julien");
@@ -137,7 +141,7 @@ public class ScmMediumTest {
   }
 
   @Test
-  public void noScmOnEmptyFile() throws IOException {
+  public void noScmOnEmptyFile() throws IOException, URISyntaxException {
 
     File baseDir = prepareProject();
 
@@ -163,7 +167,7 @@ public class ScmMediumTest {
   }
 
   @Test
-  public void log_files_with_missing_blame() throws IOException {
+  public void log_files_with_missing_blame() throws IOException, URISyntaxException {
 
     File baseDir = prepareProject();
     File xooFileWithoutBlame = new File(baseDir, "src/sample_no_blame.xoo");
@@ -194,7 +198,7 @@ public class ScmMediumTest {
 
   // SONAR-6397
   @Test
-  public void optimize_blame() throws IOException {
+  public void optimize_blame() throws IOException, URISyntaxException {
 
     File baseDir = prepareProject();
     File changedContentScmOnServer = new File(baseDir, CHANGED_CONTENT_SCM_ON_SERVER_XOO);
@@ -252,7 +256,7 @@ public class ScmMediumTest {
   }
 
   @Test
-  public void forceReload() throws IOException {
+  public void forceReload() throws IOException, URISyntaxException {
 
     File baseDir = prepareProject();
     File xooFileNoScm = new File(baseDir, SAME_CONTENT_SCM_ON_SERVER_XOO);
@@ -287,7 +291,7 @@ public class ScmMediumTest {
   }
 
   @Test
-  public void configureUsingScmURL() throws IOException {
+  public void configureUsingScmURL() throws IOException, URISyntaxException {
 
     File baseDir = prepareProject();
 
@@ -309,7 +313,7 @@ public class ScmMediumTest {
   }
 
   @Test
-  public void testAutoDetection() throws IOException {
+  public void testAutoDetection() throws IOException, URISyntaxException {
 
     File baseDir = prepareProject();
     new File(baseDir, ".xoo").createNewFile();
@@ -330,7 +334,12 @@ public class ScmMediumTest {
     assertThat(file1Scm).isNotNull();
   }
 
-  private File prepareProject() throws IOException {
+  private String getNonAsciiAuthor() throws URISyntaxException {
+    return Files.contentOf(new File(this.getClass().getResource("/mediumtest/blameAuthor.txt").toURI()), StandardCharsets.UTF_8);
+
+  }
+
+  private File prepareProject() throws IOException, URISyntaxException {
     File baseDir = temp.getRoot();
     File srcDir = new File(baseDir, "src");
     srcDir.mkdir();
@@ -341,16 +350,17 @@ public class ScmMediumTest {
     FileUtils.write(xooScmFile1,
       // revision,author,dateTime
       "1,,2013-01-04\n" +
-        "2,julien,2013-01-04\n" +
+        "2," + getNonAsciiAuthor() + ",2013-01-04\n" +
         "3,julien,2013-02-03\n" +
         "3,julien,2013-02-03\n" +
-        "4,simon,2013-03-04\n");
+        "4,simon,2013-03-04\n",
+      StandardCharsets.UTF_8);
 
     return baseDir;
   }
 
   @Test
-  public void testDisableScmSensor() throws IOException {
+  public void testDisableScmSensor() throws IOException, URISyntaxException {
 
     File baseDir = prepareProject();
 
diff --git a/sonar-scanner-engine/src/test/resources/mediumtest/blameAuthor.txt b/sonar-scanner-engine/src/test/resources/mediumtest/blameAuthor.txt
new file mode 100644
index 00000000000..4b009ad0de6
--- /dev/null
+++ b/sonar-scanner-engine/src/test/resources/mediumtest/blameAuthor.txt
@@ -0,0 +1 @@
+Name 传/傳
\ No newline at end of file
diff --git a/sonar-scanner-engine/src/test/resources/org/sonar/batch/scan/ProjectReactorBuilderTest/simple-project-with-missing-projectName/sonar-project.properties b/sonar-scanner-engine/src/test/resources/org/sonar/batch/scan/ProjectReactorBuilderTest/simple-project-with-missing-projectName/sonar-project.properties
deleted file mode 100644
index fdfffa678aa..00000000000
--- a/sonar-scanner-engine/src/test/resources/org/sonar/batch/scan/ProjectReactorBuilderTest/simple-project-with-missing-projectName/sonar-project.properties
+++ /dev/null
@@ -1,5 +0,0 @@
-sonar.projectKey=com.foo.project
-sonar.projectVersion=1.0-SNAPSHOT
-sonar.projectDescription=Description of Foo Project
-sonar.sources=.
-