SONAR-6277 Fix incorrect offset conversion with old Mac line ends '\r'

5 files changed, 139 insertions, 29 deletions

diff --git a/sonar-batch/src/test/java/org/sonar/batch/source/CodeColorizersTest.java b/sonar-batch/src/test/java/org/sonar/batch/source/CodeColorizersTest.java
index 7b56b2afb04..234d5a3c92d 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/source/CodeColorizersTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/source/CodeColorizersTest.java
@@ -20,25 +20,33 @@
 package org.sonar.batch.source;
 
 import com.google.common.collect.ImmutableList;
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 import org.apache.commons.io.FileUtils;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.api.batch.fs.internal.FileMetadata;
 import org.sonar.api.batch.sensor.highlighting.NewHighlighting;
 import org.sonar.api.batch.sensor.highlighting.TypeOfText;
+import org.sonar.api.batch.sensor.highlighting.internal.DefaultHighlighting;
+import org.sonar.api.batch.sensor.internal.SensorStorage;
 import org.sonar.api.web.CodeColorizerFormat;
 import org.sonar.colorizer.CDocTokenizer;
 import org.sonar.colorizer.CppDocTokenizer;
 import org.sonar.colorizer.JavadocTokenizer;
 import org.sonar.colorizer.KeywordsTokenizer;
+import org.sonar.colorizer.MultilinesDocTokenizer;
+import org.sonar.colorizer.RegexpTokenizer;
 import org.sonar.colorizer.StringTokenizer;
 import org.sonar.colorizer.Tokenizer;
 
-import java.io.File;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-import java.util.List;
-
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.tuple;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 
@@ -49,7 +57,7 @@ public class CodeColorizersTest {
 
   @Test
   public void testConvertToHighlighting() throws Exception {
-    CodeColorizers codeColorizers = new CodeColorizers(Arrays.<CodeColorizerFormat>asList(new JavaScriptColorizerFormat()));
+    CodeColorizers codeColorizers = new CodeColorizers(Arrays.<CodeColorizerFormat>asList(new JavaScriptColorizerFormat(), new WebCodeColorizerFormat()));
     File jsFile = new File(this.getClass().getResource("CodeColorizersTest/Person.js").toURI());
     NewHighlighting highlighting = mock(NewHighlighting.class);
 
@@ -73,7 +81,7 @@ public class CodeColorizersTest {
 
   @Test
   public void testConvertToHighlightingIgnoreBOM() throws Exception {
-    CodeColorizers codeColorizers = new CodeColorizers(Arrays.<CodeColorizerFormat>asList(new JavaScriptColorizerFormat()));
+    CodeColorizers codeColorizers = new CodeColorizers(Arrays.<CodeColorizerFormat>asList(new JavaScriptColorizerFormat(), new WebCodeColorizerFormat()));
 
     File fileWithBom = temp.newFile();
     FileUtils.write(fileWithBom, "\uFEFF", "UTF-8");
@@ -108,7 +116,27 @@ public class CodeColorizersTest {
     verify(highlighting).highlight(97, 100, TypeOfText.KEYWORD);
     verify(highlighting).highlight(142, 146, TypeOfText.KEYWORD);
     verify(highlighting).highlight(162, 170, TypeOfText.COMMENT);
+  }
 
+  @Test
+  public void testConvertHtmlToHighlightingWithMacEoL() throws Exception {
+    CodeColorizers codeColorizers = new CodeColorizers(Arrays.<CodeColorizerFormat>asList(new JavaScriptColorizerFormat(), new WebCodeColorizerFormat()));
+    File htmlFile = new File(this.getClass().getResource("CodeColorizersTest/package.html").toURI());
+    SensorStorage sensorStorage = mock(SensorStorage.class);
+    DefaultHighlighting highlighting = new DefaultHighlighting(sensorStorage);
+    highlighting.onFile(new DefaultInputFile("FOO", "package.html")
+      .initMetadata(new FileMetadata().readMetadata(htmlFile, StandardCharsets.UTF_8)));
+
+    codeColorizers.toSyntaxHighlighting(htmlFile, StandardCharsets.UTF_8, "web", highlighting);
+
+    assertThat(highlighting.getSyntaxHighlightingRuleSet()).extracting("range.start.line", "range.start.lineOffset", "range.end.line", "range.end.lineOffset", "textType")
+      .containsExactly(
+        tuple(1, 0, 1, 132, TypeOfText.STRUCTURED_COMMENT),
+        tuple(2, 0, 2, 6, TypeOfText.KEYWORD),
+        tuple(3, 0, 3, 3, TypeOfText.KEYWORD),
+        tuple(4, 0, 4, 3, TypeOfText.KEYWORD),
+        // SONARWEB-26
+        tuple(5, 42, 12, 0, TypeOfText.STRING));
   }
 
   public static class JavaScriptColorizerFormat extends CodeColorizerFormat {
@@ -165,4 +193,38 @@ public class CodeColorizersTest {
 
   }
 
+  public class WebCodeColorizerFormat extends CodeColorizerFormat {
+
+    private final List<Tokenizer> tokenizers = new ArrayList<Tokenizer>();
+
+    public WebCodeColorizerFormat() {
+      super("web");
+      String tagAfter = "</span>";
+
+      // == tags ==
+      tokenizers.add(new RegexpTokenizer("<span class=\"k\">", tagAfter, "</?[:\\w]+>?"));
+      tokenizers.add(new RegexpTokenizer("<span class=\"k\">", tagAfter, ">"));
+
+      // == doctype ==
+      tokenizers.add(new RegexpTokenizer("<span class=\"j\">", tagAfter, "<!DOCTYPE.*>"));
+
+      // == comments ==
+      tokenizers.add(new MultilinesDocTokenizer("<!--", "-->", "<span class=\"j\">", tagAfter));
+      tokenizers.add(new MultilinesDocTokenizer("<%--", "--%>", "<span class=\"j\">", tagAfter));
+
+      // == expressions ==
+      tokenizers.add(new MultilinesDocTokenizer("<%@", "%>", "<span class=\"a\">", tagAfter));
+      tokenizers.add(new MultilinesDocTokenizer("<%", "%>", "<span class=\"a\">", tagAfter));
+
+      // == tag properties ==
+      tokenizers.add(new StringTokenizer("<span class=\"s\">", tagAfter));
+    }
+
+    @Override
+    public List<Tokenizer> getTokenizers() {
+      return tokenizers;
+    }
+
+  }
+
 }
diff --git a/sonar-batch/src/test/resources/org/sonar/batch/source/CodeColorizersTest/package.html b/sonar-batch/src/test/resources/org/sonar/batch/source/CodeColorizersTest/package.html
new file mode 100644
index 00000000000..f2d90e627d6
--- /dev/null
+++ b/sonar-batch/src/test/resources/org/sonar/batch/source/CodeColorizersTest/package.html
@@ -0,0 +1 @@
+<!-- $Header: /cvshome/build/org.osgi.service.log/src/org/osgi/service/log/package.html,v 1.3 2005/08/10 01:43:20 hargrave Exp $ -->
<BODY>
<P>The OSGi Log Service Package. Specification Version 1.3.
<p>Bundles wishing to use this package must list the package
in the Import-Package header of the bundle's manifest.
For example:
<pre>
Import-Package: org.osgi.service.log; version=1.3
</pre>
</BODY>

\ No newline at end of file
diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/FileMetadata.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/FileMetadata.java
index 8d449f4ab4c..aa275e19b28 100644
--- a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/FileMetadata.java
+++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/FileMetadata.java
@@ -20,18 +20,6 @@
 package org.sonar.api.batch.fs.internal;
 
 import com.google.common.primitives.Ints;
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.io.ByteOrderMark;
-import org.apache.commons.io.input.BOMInputStream;
-import org.sonar.api.batch.BatchSide;
-import org.sonar.api.CoreProperties;
-import org.sonar.api.utils.log.Logger;
-import org.sonar.api.utils.log.Loggers;
-
-import javax.annotation.CheckForNull;
-import javax.annotation.Nullable;
-
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;
@@ -48,6 +36,16 @@ import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
 import java.util.ArrayList;
 import java.util.List;
+import javax.annotation.CheckForNull;
+import javax.annotation.Nullable;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.io.ByteOrderMark;
+import org.apache.commons.io.input.BOMInputStream;
+import org.sonar.api.CoreProperties;
+import org.sonar.api.batch.BatchSide;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
 
 /**
  * Computes hash of files. Ends of Lines are ignored, so files with
@@ -311,11 +309,19 @@ public class FileMetadata {
       c = (char) i;
       if (afterCR) {
         for (CharHandler handler : handlers) {
-          if (c != CARRIAGE_RETURN && c != LINE_FEED) {
+          if (c == CARRIAGE_RETURN) {
+            handler.newLine();
+            handler.handleAll(c);
+          } else if (c == LINE_FEED) {
+            handler.handleAll(c);
+            handler.newLine();
+            afterCR = false;
+          } else {
+            handler.newLine();
             handler.handleIgnoreEoL(c);
+            handler.handleAll(c);
+            afterCR = false;
           }
-          handler.handleAll(c);
-          handler.newLine();
         }
         afterCR = c == CARRIAGE_RETURN;
       } else if (c == LINE_FEED) {
@@ -337,6 +343,9 @@ public class FileMetadata {
       i = reader.read();
     }
     for (CharHandler handler : handlers) {
+      if (afterCR) {
+        handler.newLine();
+      }
       handler.eof();
     }
   }
diff --git a/sonar-plugin-api/src/test/java/org/sonar/api/batch/fs/internal/DefaultInputFileTest.java b/sonar-plugin-api/src/test/java/org/sonar/api/batch/fs/internal/DefaultInputFileTest.java
index 99f70ac964b..f5792ad178d 100644
--- a/sonar-plugin-api/src/test/java/org/sonar/api/batch/fs/internal/DefaultInputFileTest.java
+++ b/sonar-plugin-api/src/test/java/org/sonar/api/batch/fs/internal/DefaultInputFileTest.java
@@ -25,6 +25,7 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.TextRange;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.fail;
@@ -179,6 +180,19 @@ public class DefaultInputFileTest {
   }
 
   @Test
+  public void checkValidRangeUsingGlobalOffset() {
+    DefaultInputFile file = new DefaultInputFile("ABCDE", "src/Foo.php");
+    file.setLines(2);
+    file.setOriginalLineOffsets(new int[] {0, 10});
+    file.setLastValidOffset(15);
+    TextRange newRange = file.newRange(10, 13);
+    assertThat(newRange.start().line()).isEqualTo(2);
+    assertThat(newRange.start().lineOffset()).isEqualTo(0);
+    assertThat(newRange.end().line()).isEqualTo(2);
+    assertThat(newRange.end().lineOffset()).isEqualTo(3);
+  }
+
+  @Test
   public void testRangeOverlap() {
     DefaultInputFile file = new DefaultInputFile("ABCDE", "src/Foo.php");
     file.setLines(2);
diff --git a/sonar-plugin-api/src/test/java/org/sonar/api/batch/fs/internal/FileMetadataTest.java b/sonar-plugin-api/src/test/java/org/sonar/api/batch/fs/internal/FileMetadataTest.java
index 9dbd8442527..7cd43d46423 100644
--- a/sonar-plugin-api/src/test/java/org/sonar/api/batch/fs/internal/FileMetadataTest.java
+++ b/sonar-plugin-api/src/test/java/org/sonar/api/batch/fs/internal/FileMetadataTest.java
@@ -19,6 +19,10 @@
  */
 package org.sonar.api.batch.fs.internal;
 
+import java.io.File;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import javax.annotation.Nullable;
 import org.apache.commons.codec.binary.Hex;
 import org.apache.commons.io.FileUtils;
 import org.junit.Rule;
@@ -29,12 +33,6 @@ import org.sonar.api.batch.fs.internal.FileMetadata.LineHashConsumer;
 import org.sonar.api.utils.log.LogTester;
 import org.sonar.api.utils.log.LoggerLevel;
 
-import javax.annotation.Nullable;
-
-import java.io.File;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-
 import static org.apache.commons.codec.digest.DigestUtils.md5Hex;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.fail;
@@ -137,6 +135,32 @@ public class FileMetadataTest {
   }
 
   @Test
+  public void mac_without_latest_eol() throws Exception {
+    File tempFile = temp.newFile();
+    FileUtils.write(tempFile, "foo\rbar\rbaz", StandardCharsets.UTF_8, true);
+
+    FileMetadata.Metadata metadata = new FileMetadata().readMetadata(tempFile, StandardCharsets.UTF_8);
+    assertThat(metadata.lines).isEqualTo(3);
+    assertThat(metadata.nonBlankLines).isEqualTo(3);
+    assertThat(metadata.hash).isEqualTo(md5Hex("foo\nbar\nbaz"));
+    assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 8);
+    assertThat(metadata.lastValidOffset).isEqualTo(11);
+  }
+
+  @Test
+  public void mac_with_latest_eol() throws Exception {
+    File tempFile = temp.newFile();
+    FileUtils.write(tempFile, "foo\rbar\rbaz\r", StandardCharsets.UTF_8, true);
+
+    FileMetadata.Metadata metadata = new FileMetadata().readMetadata(tempFile, StandardCharsets.UTF_8);
+    assertThat(metadata.lines).isEqualTo(4);
+    assertThat(metadata.nonBlankLines).isEqualTo(3);
+    assertThat(metadata.hash).isEqualTo(md5Hex("foo\nbar\nbaz\n"));
+    assertThat(metadata.originalLineOffsets).containsOnly(0, 4, 8, 12);
+    assertThat(metadata.lastValidOffset).isEqualTo(12);
+  }
+
+  @Test
   public void mix_of_newlines_with_latest_eol() throws Exception {
     File tempFile = temp.newFile();
     FileUtils.write(tempFile, "foo\nbar\r\nbaz\n", StandardCharsets.UTF_8, true);
@@ -284,7 +308,7 @@ public class FileMetadataTest {
 
     FileMetadata.Metadata metadata = new FileMetadata().readMetadata(woff, StandardCharsets.UTF_8);
     assertThat(metadata.lines).isEqualTo(135);
-    assertThat(metadata.nonBlankLines).isEqualTo(134);
+    assertThat(metadata.nonBlankLines).isEqualTo(133);
     assertThat(metadata.hash).isNotEmpty();
 
     assertThat(logTester.logs(LoggerLevel.WARN).get(0)).contains("Invalid character encountered in file");