SONAR-3531 Allow to use MultilinesDocTokenizer for html comments

3 files changed, 43 insertions, 30 deletions

diff --git a/sonar-colorizer/src/main/java/org/sonar/colorizer/MultilinesDocTokenizer.java b/sonar-colorizer/src/main/java/org/sonar/colorizer/MultilinesDocTokenizer.java
index 12795af506f..5471bcc4fc8 100644
--- a/sonar-colorizer/src/main/java/org/sonar/colorizer/MultilinesDocTokenizer.java
+++ b/sonar-colorizer/src/main/java/org/sonar/colorizer/MultilinesDocTokenizer.java
@@ -19,11 +19,11 @@
  */
 package org.sonar.colorizer;
 
-import java.util.Arrays;
-
 import org.sonar.channel.CodeReader;
 import org.sonar.channel.EndMatcher;
 
+import java.util.Arrays;
+
 public class MultilinesDocTokenizer extends Tokenizer {
 
   private static final String COMMENT_STARTED_ON_PREVIOUS_LINE = "COMMENT_STARTED_ON_PREVIOUS_LINE";
@@ -59,7 +59,7 @@ public class MultilinesDocTokenizer extends Tokenizer {
   public boolean consume(CodeReader code, HtmlCodeBuilder codeBuilder) {
     if (hasNextToken(code, codeBuilder)) {
       codeBuilder.appendWithoutTransforming(tagBefore);
-      code.popTo(new MultilineEndTokenMatcher(codeBuilder), codeBuilder);
+      code.popTo(new MultilineEndTokenMatcher(code, codeBuilder), codeBuilder);
       codeBuilder.appendWithoutTransforming(tagAfter);
       return true;
     } else {
@@ -69,33 +69,33 @@ public class MultilinesDocTokenizer extends Tokenizer {
 
   private class MultilineEndTokenMatcher implements EndMatcher {
 
-    private final HtmlCodeBuilder code;
-    private final StringBuilder colorizedCode;
+    private final CodeReader code;
+    private final HtmlCodeBuilder codeBuilder;
     private int commentSize = 0;
 
-    public MultilineEndTokenMatcher(HtmlCodeBuilder code) {
+    public MultilineEndTokenMatcher(CodeReader code, HtmlCodeBuilder codeBuilder) {
       this.code = code;
-      this.colorizedCode = code.getColorizedCode();
+      this.codeBuilder = codeBuilder;
     }
 
     public boolean match(int endFlag) {
       commentSize++;
-      if (commentSize >= endToken.length + startToken.length || (commentSize >= endToken.length && isCommentStartedOnPreviousLine(code))) {
+      if (commentSize >= endToken.length + startToken.length || (commentSize >= endToken.length && isCommentStartedOnPreviousLine(codeBuilder))) {
         boolean matches = true;
         for (int i = 1; i <= endToken.length; i++) {
-          if (colorizedCode.charAt(colorizedCode.length() - i) != endToken[endToken.length - i]) {
+          if (code.charAt(-i) != endToken[endToken.length - i]) {
             matches = false;
             break;
           }
         }
         if (matches) {
-          setCommentStartedOnPreviousLine(code, Boolean.FALSE);
+          setCommentStartedOnPreviousLine(codeBuilder, Boolean.FALSE);
           return true;
         }
       }
 
       if (endFlag == '\r' || endFlag == '\n') {
-        setCommentStartedOnPreviousLine(code, Boolean.TRUE);
+        setCommentStartedOnPreviousLine(codeBuilder, Boolean.TRUE);
         return true;
       }
       return false;
@@ -115,4 +115,5 @@ public class MultilinesDocTokenizer extends Tokenizer {
   private String getTokenizerId() {
     return getClass().getSimpleName();
   }
+
 }
diff --git a/sonar-colorizer/src/main/java/org/sonar/colorizer/Tokenizer.java b/sonar-colorizer/src/main/java/org/sonar/colorizer/Tokenizer.java
index 59a6eed3b3a..7def41c4e9a 100644
--- a/sonar-colorizer/src/main/java/org/sonar/colorizer/Tokenizer.java
+++ b/sonar-colorizer/src/main/java/org/sonar/colorizer/Tokenizer.java
@@ -24,8 +24,8 @@ import org.sonar.channel.Channel;
 /**
  * A token highlighter is in charge to optionally consume the next characters (token) of a CodeReader
  * and to add the highlighted token to an HtmlCodeBuilder.
- * 
- * IMPORTANT: a Tokenizer must be THREAD-SAFE otherwise see NotThreadSafeTokenizer
+ *
+ * IMPORTANT: a Tokenizer must be THREAD-SAFE otherwise use {@link NotThreadSafeTokenizer}
  */
 public abstract class Tokenizer extends Channel<HtmlCodeBuilder> {
 }
diff --git a/sonar-colorizer/src/test/java/org/sonar/colorizer/MultilinesDocTokenizerTest.java b/sonar-colorizer/src/test/java/org/sonar/colorizer/MultilinesDocTokenizerTest.java
index c68fe773c1a..8265efae16c 100644
--- a/sonar-colorizer/src/test/java/org/sonar/colorizer/MultilinesDocTokenizerTest.java
+++ b/sonar-colorizer/src/test/java/org/sonar/colorizer/MultilinesDocTokenizerTest.java
@@ -24,10 +24,7 @@ import org.junit.Before;
 import org.junit.Test;
 import org.sonar.channel.CodeReader;
 
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.is;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import static org.fest.assertions.Assertions.assertThat;
 
 public class MultilinesDocTokenizerTest {
 
@@ -41,37 +38,52 @@ public class MultilinesDocTokenizerTest {
   @Test
   public void testStandardComment() {
     MultilinesDocTokenizer tokenizer = new MultiLineDocTokenizerImpl("{[||", "");
-    assertThat(tokenizer.hasNextToken(new CodeReader("{[|| And here is strange  multi-line comment"), new HtmlCodeBuilder()), is(true));
-    assertThat(tokenizer.hasNextToken(new CodeReader("// this is not a strange multi-line comment"), new HtmlCodeBuilder()), is(false));
+    assertThat(tokenizer.hasNextToken(new CodeReader("{[|| And here is strange  multi-line comment"), new HtmlCodeBuilder())).isTrue();
+    assertThat(tokenizer.hasNextToken(new CodeReader("// this is not a strange multi-line comment"), new HtmlCodeBuilder())).isFalse();
   }
 
   @Test
   public void testLongStartToken() {
+    CodeReader reader = new CodeReader("/*** multi-line comment**/ private part");
     MultilinesDocTokenizer tokenizer = new MultiLineDocTokenizerImpl("/***", "**/");
-    assertTrue(tokenizer.consume(new CodeReader("/*** multi-line comment**/ private part"), codeBuilder));
-    assertEquals("/*** multi-line comment**/", codeBuilder.toString());
+    assertThat(tokenizer.consume(reader, codeBuilder)).isTrue();
+    assertThat(codeBuilder.toString()).isEqualTo("/*** multi-line comment**/");
   }
-  
+
   @Test
   public void testStartTokenEndTokenOverlapping() {
+    CodeReader reader = new CodeReader("/*// multi-line comment*/ private part");
     MultilinesDocTokenizer tokenizer = new MultiLineDocTokenizerImpl("/*", "*/");
-    assertTrue(tokenizer.consume(new CodeReader("/*// multi-line comment*/ private part"), codeBuilder));
-    assertEquals("/*// multi-line comment*/", codeBuilder.toString());
+    assertThat(tokenizer.consume(reader, codeBuilder)).isTrue();
+    assertThat(codeBuilder.toString()).isEqualTo("/*// multi-line comment*/");
   }
-  
+
   @Test
   public void testMultilinesComment() {
     CodeReader reader = new CodeReader("/* multi-line comment\n*/ private part");
     MultilinesDocTokenizer tokenizer = new MultiLineDocTokenizerImpl("/*", "*/");
-    assertTrue(tokenizer.consume(reader, codeBuilder));
-    assertEquals("/* multi-line comment", codeBuilder.toString());
+    assertThat(tokenizer.consume(reader, codeBuilder)).isTrue();
+    assertThat(codeBuilder.toString()).isEqualTo("/* multi-line comment");
     reader.pop();
-    assertTrue(tokenizer.consume(reader, codeBuilder));
-    assertEquals("/* multi-line comment*/", codeBuilder.toString());
+    assertThat(tokenizer.consume(reader, codeBuilder)).isTrue();
+    assertThat(codeBuilder.toString()).isEqualTo("/* multi-line comment*/");
   }
 
-  public class MultiLineDocTokenizerImpl extends MultilinesDocTokenizer {
+  /**
+   * SONAR-3531
+   */
+  @Test
+  public void should_work_for_html_comments() {
+    CodeReader reader = new CodeReader("<!-- multi-line comment\n--> private part");
+    MultilinesDocTokenizer tokenizer = new MultiLineDocTokenizerImpl("<!--", "-->");
+    assertThat(tokenizer.consume(reader, codeBuilder)).isTrue();
+    assertThat(codeBuilder.toString()).isEqualTo("&lt;!-- multi-line comment");
+    reader.pop();
+    assertThat(tokenizer.consume(reader, codeBuilder)).isTrue();
+    assertThat(codeBuilder.toString()).isEqualTo("&lt;!-- multi-line comment--&gt;");
+  }
 
+  public class MultiLineDocTokenizerImpl extends MultilinesDocTokenizer {
     public MultiLineDocTokenizerImpl(String startToken, String endToken) {
       super(startToken, endToken, "", "");
     }