SONAR-3531 Allow to use MultilinesDocTokenizer for html comments

diff --git a/sonar-colorizer/pom.xml b/sonar-colorizer/pom.xml
index dc3960e8a53caa32f6de3a6dfe91a21f704e4db1..13aebf419fc6238e3fa1a0b59ccf5eceb0282abc 100644 (file)
--- a/sonar-colorizer/pom.xml
+++ b/sonar-colorizer/pom.xml
@@ -1,14 +1,16 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
+
   <parent>
     <groupId>org.codehaus.sonar</groupId>
     <artifactId>sonar</artifactId>
     <version>3.2-SNAPSHOT</version>
     <relativePath>..</relativePath>
   </parent>
+
   <artifactId>sonar-colorizer</artifactId>
-  <packaging>jar</packaging>
+
   <name>Sonar :: Code Colorizer</name>
   <description>Code syntax highlighter</description>
 
@@ -31,5 +33,11 @@
       <artifactId>hamcrest-all</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.easytesting</groupId>
+      <artifactId>fest-assert</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
-</project>
\ No newline at end of file
+
+</project>

diff --git a/sonar-colorizer/src/main/java/org/sonar/colorizer/MultilinesDocTokenizer.java b/sonar-colorizer/src/main/java/org/sonar/colorizer/MultilinesDocTokenizer.java
index 12795af506f8f4ededf93e0218b0eced20325b9a..5471bcc4fc838cba6b7ea995296af87631c45b60 100644 (file)
--- a/sonar-colorizer/src/main/java/org/sonar/colorizer/MultilinesDocTokenizer.java
+++ b/sonar-colorizer/src/main/java/org/sonar/colorizer/MultilinesDocTokenizer.java
@@ -19,11 +19,11 @@
  */
 package org.sonar.colorizer;
 
-import java.util.Arrays;
-
 import org.sonar.channel.CodeReader;
 import org.sonar.channel.EndMatcher;
 
+import java.util.Arrays;
+
 public class MultilinesDocTokenizer extends Tokenizer {
 
   private static final String COMMENT_STARTED_ON_PREVIOUS_LINE = "COMMENT_STARTED_ON_PREVIOUS_LINE";
@@ -59,7 +59,7 @@ public class MultilinesDocTokenizer extends Tokenizer {
   public boolean consume(CodeReader code, HtmlCodeBuilder codeBuilder) {
     if (hasNextToken(code, codeBuilder)) {
       codeBuilder.appendWithoutTransforming(tagBefore);
-      code.popTo(new MultilineEndTokenMatcher(codeBuilder), codeBuilder);
+      code.popTo(new MultilineEndTokenMatcher(code, codeBuilder), codeBuilder);
       codeBuilder.appendWithoutTransforming(tagAfter);
       return true;
     } else {
@@ -69,33 +69,33 @@ public class MultilinesDocTokenizer extends Tokenizer {
 
   private class MultilineEndTokenMatcher implements EndMatcher {
 
-    private final HtmlCodeBuilder code;
-    private final StringBuilder colorizedCode;
+    private final CodeReader code;
+    private final HtmlCodeBuilder codeBuilder;
     private int commentSize = 0;
 
-    public MultilineEndTokenMatcher(HtmlCodeBuilder code) {
+    public MultilineEndTokenMatcher(CodeReader code, HtmlCodeBuilder codeBuilder) {
       this.code = code;
-      this.colorizedCode = code.getColorizedCode();
+      this.codeBuilder = codeBuilder;
     }
 
     public boolean match(int endFlag) {
       commentSize++;
-      if (commentSize >= endToken.length + startToken.length || (commentSize >= endToken.length && isCommentStartedOnPreviousLine(code))) {
+      if (commentSize >= endToken.length + startToken.length || (commentSize >= endToken.length && isCommentStartedOnPreviousLine(codeBuilder))) {
         boolean matches = true;
         for (int i = 1; i <= endToken.length; i++) {
-          if (colorizedCode.charAt(colorizedCode.length() - i) != endToken[endToken.length - i]) {
+          if (code.charAt(-i) != endToken[endToken.length - i]) {
             matches = false;
             break;
           }
         }
         if (matches) {
-          setCommentStartedOnPreviousLine(code, Boolean.FALSE);
+          setCommentStartedOnPreviousLine(codeBuilder, Boolean.FALSE);
           return true;
         }
       }
 
       if (endFlag == '\r' || endFlag == '\n') {
-        setCommentStartedOnPreviousLine(code, Boolean.TRUE);
+        setCommentStartedOnPreviousLine(codeBuilder, Boolean.TRUE);
         return true;
       }
       return false;
@@ -115,4 +115,5 @@ public class MultilinesDocTokenizer extends Tokenizer {
   private String getTokenizerId() {
     return getClass().getSimpleName();
   }
+
 }

diff --git a/sonar-colorizer/src/main/java/org/sonar/colorizer/Tokenizer.java b/sonar-colorizer/src/main/java/org/sonar/colorizer/Tokenizer.java
index 59a6eed3b3a32e19b7014bbd7ea4d432b964dbf3..7def41c4e9adbe7a4043aad7becdaac63c51e618 100644 (file)
--- a/sonar-colorizer/src/main/java/org/sonar/colorizer/Tokenizer.java
+++ b/sonar-colorizer/src/main/java/org/sonar/colorizer/Tokenizer.java
@@ -24,8 +24,8 @@ import org.sonar.channel.Channel;
 /**
  * A token highlighter is in charge to optionally consume the next characters (token) of a CodeReader
  * and to add the highlighted token to an HtmlCodeBuilder.
- * 
- * IMPORTANT: a Tokenizer must be THREAD-SAFE otherwise see NotThreadSafeTokenizer
+ *
+ * IMPORTANT: a Tokenizer must be THREAD-SAFE otherwise use {@link NotThreadSafeTokenizer}
  */
 public abstract class Tokenizer extends Channel<HtmlCodeBuilder> {
 }

diff --git a/sonar-colorizer/src/test/java/org/sonar/colorizer/MultilinesDocTokenizerTest.java b/sonar-colorizer/src/test/java/org/sonar/colorizer/MultilinesDocTokenizerTest.java
index c68fe773c1a99eb6d52ace933a9777ce83d2a8dc..8265efae16c0a5b537a83eccaa4d39e81090a272 100644 (file)
--- a/sonar-colorizer/src/test/java/org/sonar/colorizer/MultilinesDocTokenizerTest.java
+++ b/sonar-colorizer/src/test/java/org/sonar/colorizer/MultilinesDocTokenizerTest.java
@@ -24,10 +24,7 @@ import org.junit.Before;
 import org.junit.Test;
 import org.sonar.channel.CodeReader;
 
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.is;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import static org.fest.assertions.Assertions.assertThat;
 
 public class MultilinesDocTokenizerTest {
 
@@ -41,37 +38,52 @@ public class MultilinesDocTokenizerTest {
   @Test
   public void testStandardComment() {
     MultilinesDocTokenizer tokenizer = new MultiLineDocTokenizerImpl("{[||", "");
-    assertThat(tokenizer.hasNextToken(new CodeReader("{[|| And here is strange  multi-line comment"), new HtmlCodeBuilder()), is(true));
-    assertThat(tokenizer.hasNextToken(new CodeReader("// this is not a strange multi-line comment"), new HtmlCodeBuilder()), is(false));
+    assertThat(tokenizer.hasNextToken(new CodeReader("{[|| And here is strange  multi-line comment"), new HtmlCodeBuilder())).isTrue();
+    assertThat(tokenizer.hasNextToken(new CodeReader("// this is not a strange multi-line comment"), new HtmlCodeBuilder())).isFalse();
   }
 
   @Test
   public void testLongStartToken() {
+    CodeReader reader = new CodeReader("/*** multi-line comment**/ private part");
     MultilinesDocTokenizer tokenizer = new MultiLineDocTokenizerImpl("/***", "**/");
-    assertTrue(tokenizer.consume(new CodeReader("/*** multi-line comment**/ private part"), codeBuilder));
-    assertEquals("/*** multi-line comment**/", codeBuilder.toString());
+    assertThat(tokenizer.consume(reader, codeBuilder)).isTrue();
+    assertThat(codeBuilder.toString()).isEqualTo("/*** multi-line comment**/");
   }
-  
+
   @Test
   public void testStartTokenEndTokenOverlapping() {
+    CodeReader reader = new CodeReader("/*// multi-line comment*/ private part");
     MultilinesDocTokenizer tokenizer = new MultiLineDocTokenizerImpl("/*", "*/");
-    assertTrue(tokenizer.consume(new CodeReader("/*// multi-line comment*/ private part"), codeBuilder));
-    assertEquals("/*// multi-line comment*/", codeBuilder.toString());
+    assertThat(tokenizer.consume(reader, codeBuilder)).isTrue();
+    assertThat(codeBuilder.toString()).isEqualTo("/*// multi-line comment*/");
   }
-  
+
   @Test
   public void testMultilinesComment() {
     CodeReader reader = new CodeReader("/* multi-line comment\n*/ private part");
     MultilinesDocTokenizer tokenizer = new MultiLineDocTokenizerImpl("/*", "*/");
-    assertTrue(tokenizer.consume(reader, codeBuilder));
-    assertEquals("/* multi-line comment", codeBuilder.toString());
+    assertThat(tokenizer.consume(reader, codeBuilder)).isTrue();
+    assertThat(codeBuilder.toString()).isEqualTo("/* multi-line comment");
     reader.pop();
-    assertTrue(tokenizer.consume(reader, codeBuilder));
-    assertEquals("/* multi-line comment*/", codeBuilder.toString());
+    assertThat(tokenizer.consume(reader, codeBuilder)).isTrue();
+    assertThat(codeBuilder.toString()).isEqualTo("/* multi-line comment*/");
   }
 
-  public class MultiLineDocTokenizerImpl extends MultilinesDocTokenizer {
+  /**
+   * SONAR-3531
+   */
+  @Test
+  public void should_work_for_html_comments() {
+    CodeReader reader = new CodeReader("<!-- multi-line comment\n--> private part");
+    MultilinesDocTokenizer tokenizer = new MultiLineDocTokenizerImpl("<!--", "-->");
+    assertThat(tokenizer.consume(reader, codeBuilder)).isTrue();
+    assertThat(codeBuilder.toString()).isEqualTo("&lt;!-- multi-line comment");
+    reader.pop();
+    assertThat(tokenizer.consume(reader, codeBuilder)).isTrue();
+    assertThat(codeBuilder.toString()).isEqualTo("&lt;!-- multi-line comment--&gt;");
+  }
 
+  public class MultiLineDocTokenizerImpl extends MultilinesDocTokenizer {
     public MultiLineDocTokenizerImpl(String startToken, String endToken) {
       super(startToken, endToken, "", "");
     }