SONAR-6277 Fix incorrect offset conversion with old Mac line ends '\r'

2 files changed, 70 insertions, 7 deletions

diff --git a/sonar-batch/src/test/java/org/sonar/batch/source/CodeColorizersTest.java b/sonar-batch/src/test/java/org/sonar/batch/source/CodeColorizersTest.java
index 7b56b2afb04..234d5a3c92d 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/source/CodeColorizersTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/source/CodeColorizersTest.java
@@ -20,25 +20,33 @@
 package org.sonar.batch.source;
 
 import com.google.common.collect.ImmutableList;
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 import org.apache.commons.io.FileUtils;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.api.batch.fs.internal.FileMetadata;
 import org.sonar.api.batch.sensor.highlighting.NewHighlighting;
 import org.sonar.api.batch.sensor.highlighting.TypeOfText;
+import org.sonar.api.batch.sensor.highlighting.internal.DefaultHighlighting;
+import org.sonar.api.batch.sensor.internal.SensorStorage;
 import org.sonar.api.web.CodeColorizerFormat;
 import org.sonar.colorizer.CDocTokenizer;
 import org.sonar.colorizer.CppDocTokenizer;
 import org.sonar.colorizer.JavadocTokenizer;
 import org.sonar.colorizer.KeywordsTokenizer;
+import org.sonar.colorizer.MultilinesDocTokenizer;
+import org.sonar.colorizer.RegexpTokenizer;
 import org.sonar.colorizer.StringTokenizer;
 import org.sonar.colorizer.Tokenizer;
 
-import java.io.File;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-import java.util.List;
-
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.tuple;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 
@@ -49,7 +57,7 @@ public class CodeColorizersTest {
 
   @Test
   public void testConvertToHighlighting() throws Exception {
-    CodeColorizers codeColorizers = new CodeColorizers(Arrays.<CodeColorizerFormat>asList(new JavaScriptColorizerFormat()));
+    CodeColorizers codeColorizers = new CodeColorizers(Arrays.<CodeColorizerFormat>asList(new JavaScriptColorizerFormat(), new WebCodeColorizerFormat()));
     File jsFile = new File(this.getClass().getResource("CodeColorizersTest/Person.js").toURI());
     NewHighlighting highlighting = mock(NewHighlighting.class);
 
@@ -73,7 +81,7 @@ public class CodeColorizersTest {
 
   @Test
   public void testConvertToHighlightingIgnoreBOM() throws Exception {
-    CodeColorizers codeColorizers = new CodeColorizers(Arrays.<CodeColorizerFormat>asList(new JavaScriptColorizerFormat()));
+    CodeColorizers codeColorizers = new CodeColorizers(Arrays.<CodeColorizerFormat>asList(new JavaScriptColorizerFormat(), new WebCodeColorizerFormat()));
 
     File fileWithBom = temp.newFile();
     FileUtils.write(fileWithBom, "\uFEFF", "UTF-8");
@@ -108,7 +116,27 @@ public class CodeColorizersTest {
     verify(highlighting).highlight(97, 100, TypeOfText.KEYWORD);
     verify(highlighting).highlight(142, 146, TypeOfText.KEYWORD);
     verify(highlighting).highlight(162, 170, TypeOfText.COMMENT);
+  }
 
+  @Test
+  public void testConvertHtmlToHighlightingWithMacEoL() throws Exception {
+    CodeColorizers codeColorizers = new CodeColorizers(Arrays.<CodeColorizerFormat>asList(new JavaScriptColorizerFormat(), new WebCodeColorizerFormat()));
+    File htmlFile = new File(this.getClass().getResource("CodeColorizersTest/package.html").toURI());
+    SensorStorage sensorStorage = mock(SensorStorage.class);
+    DefaultHighlighting highlighting = new DefaultHighlighting(sensorStorage);
+    highlighting.onFile(new DefaultInputFile("FOO", "package.html")
+      .initMetadata(new FileMetadata().readMetadata(htmlFile, StandardCharsets.UTF_8)));
+
+    codeColorizers.toSyntaxHighlighting(htmlFile, StandardCharsets.UTF_8, "web", highlighting);
+
+    assertThat(highlighting.getSyntaxHighlightingRuleSet()).extracting("range.start.line", "range.start.lineOffset", "range.end.line", "range.end.lineOffset", "textType")
+      .containsExactly(
+        tuple(1, 0, 1, 132, TypeOfText.STRUCTURED_COMMENT),
+        tuple(2, 0, 2, 6, TypeOfText.KEYWORD),
+        tuple(3, 0, 3, 3, TypeOfText.KEYWORD),
+        tuple(4, 0, 4, 3, TypeOfText.KEYWORD),
+        // SONARWEB-26
+        tuple(5, 42, 12, 0, TypeOfText.STRING));
   }
 
   public static class JavaScriptColorizerFormat extends CodeColorizerFormat {
@@ -165,4 +193,38 @@ public class CodeColorizersTest {
 
   }
 
+  public class WebCodeColorizerFormat extends CodeColorizerFormat {
+
+    private final List<Tokenizer> tokenizers = new ArrayList<Tokenizer>();
+
+    public WebCodeColorizerFormat() {
+      super("web");
+      String tagAfter = "</span>";
+
+      // == tags ==
+      tokenizers.add(new RegexpTokenizer("<span class=\"k\">", tagAfter, "</?[:\\w]+>?"));
+      tokenizers.add(new RegexpTokenizer("<span class=\"k\">", tagAfter, ">"));
+
+      // == doctype ==
+      tokenizers.add(new RegexpTokenizer("<span class=\"j\">", tagAfter, "<!DOCTYPE.*>"));
+
+      // == comments ==
+      tokenizers.add(new MultilinesDocTokenizer("<!--", "-->", "<span class=\"j\">", tagAfter));
+      tokenizers.add(new MultilinesDocTokenizer("<%--", "--%>", "<span class=\"j\">", tagAfter));
+
+      // == expressions ==
+      tokenizers.add(new MultilinesDocTokenizer("<%@", "%>", "<span class=\"a\">", tagAfter));
+      tokenizers.add(new MultilinesDocTokenizer("<%", "%>", "<span class=\"a\">", tagAfter));
+
+      // == tag properties ==
+      tokenizers.add(new StringTokenizer("<span class=\"s\">", tagAfter));
+    }
+
+    @Override
+    public List<Tokenizer> getTokenizers() {
+      return tokenizers;
+    }
+
+  }
+
 }
diff --git a/sonar-batch/src/test/resources/org/sonar/batch/source/CodeColorizersTest/package.html b/sonar-batch/src/test/resources/org/sonar/batch/source/CodeColorizersTest/package.html
new file mode 100644
index 00000000000..f2d90e627d6
--- /dev/null
+++ b/sonar-batch/src/test/resources/org/sonar/batch/source/CodeColorizersTest/package.html
@@ -0,0 +1 @@
+<!-- $Header: /cvshome/build/org.osgi.service.log/src/org/osgi/service/log/package.html,v 1.3 2005/08/10 01:43:20 hargrave Exp $ -->
<BODY>
<P>The OSGi Log Service Package. Specification Version 1.3.
<p>Bundles wishing to use this package must list the package
in the Import-Package header of the bundle's manifest.
For example:
<pre>
Import-Package: org.osgi.service.log; version=1.3
</pre>
</BODY>

\ No newline at end of file