SONAR-236 remove deprecated code from checkstyle plugin + display default value of rule parameters in Q profile console

23 files changed, 1253 insertions, 0 deletions

diff --git a/sonar-colorizer/src/test/java/org/sonar/colorizer/CDocTokenizerTest.java b/sonar-colorizer/src/test/java/org/sonar/colorizer/CDocTokenizerTest.java
new file mode 100644
index 00000000000..31b7e316de4
--- /dev/null
+++ b/sonar-colorizer/src/test/java/org/sonar/colorizer/CDocTokenizerTest.java
@@ -0,0 +1,49 @@
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2009 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
+ */
+package org.sonar.colorizer;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.sonar.channel.CodeReader;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+public class CDocTokenizerTest {
+
+  private HtmlCodeBuilder codeBuilder;
+
+  @Before
+  public void init() {
+    codeBuilder = new HtmlCodeBuilder();
+  }
+
+  @Test
+  public void testRead() {
+    CDocTokenizer tokenizer = new CDocTokenizer("<c>", "</c>");
+    assertTrue(tokenizer.consume(new CodeReader("//this is a comment"), codeBuilder));
+    assertThat(codeBuilder.toString(), is("<c>//this is a comment</c>"));
+
+    assertFalse(tokenizer.consume(new CodeReader("this is not a comment"), codeBuilder));
+    assertThat(codeBuilder.toString(), is("<c>//this is a comment</c>"));
+  }
+}
diff --git a/sonar-colorizer/src/test/java/org/sonar/colorizer/CaseInsensitiveKeywordsTokenizerTest.java b/sonar-colorizer/src/test/java/org/sonar/colorizer/CaseInsensitiveKeywordsTokenizerTest.java
new file mode 100644
index 00000000000..8f6d7412b91
--- /dev/null
+++ b/sonar-colorizer/src/test/java/org/sonar/colorizer/CaseInsensitiveKeywordsTokenizerTest.java
@@ -0,0 +1,46 @@
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2009 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
+ */
+package org.sonar.colorizer;
+
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+import static org.junit.Assert.assertThat;
+import static org.sonar.colorizer.SyntaxHighlighterTestingHarness.highlight;
+
+import org.junit.Test;
+
+public class CaseInsensitiveKeywordsTokenizerTest {
+
+  @Test
+  public void hasNextToken() {
+    CaseInsensitiveKeywordsTokenizer tokenizer = new CaseInsensitiveKeywordsTokenizer("<k>", "</k>", "PROCEDURE");
+    assertThat(highlight("procedure name", tokenizer), is("<k>procedure</k> name"));
+    assertThat(highlight("Procedure name", tokenizer), is("<k>Procedure</k> name"));
+    assertThat(highlight("PROCEDURE name", tokenizer), is("<k>PROCEDURE</k> name"));
+  }
+
+  @Test
+  public void testClone() {
+    CaseInsensitiveKeywordsTokenizer tokenizer = new CaseInsensitiveKeywordsTokenizer("<k>", "</k>", "PROCEDURE");
+    Tokenizer cloneTokenizer = tokenizer.clone();
+    assertThat(tokenizer, is(not(cloneTokenizer)));
+    assertThat(highlight("procedure name", cloneTokenizer), is("<k>procedure</k> name"));
+  }
+}
diff --git a/sonar-colorizer/src/test/java/org/sonar/colorizer/CodeColorizerTest.java b/sonar-colorizer/src/test/java/org/sonar/colorizer/CodeColorizerTest.java
new file mode 100644
index 00000000000..0c5cd743f53
--- /dev/null
+++ b/sonar-colorizer/src/test/java/org/sonar/colorizer/CodeColorizerTest.java
@@ -0,0 +1,129 @@
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2009 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
+ */
+package org.sonar.colorizer;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsNot.not;
+import static org.hamcrest.number.OrderingComparisons.greaterThan;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.Test;
+
+public class CodeColorizerTest {
+
+  @Test
+  public void javaToHtml() throws IOException {
+    Reader java = readFile("/org/sonar/colorizer/samples/Sample.java");
+
+    String html = CodeColorizer.javaToHtml(java, HtmlOptions.DEFAULT);
+
+    assertHtml(html);
+    assertContains(html, "<pre><span class=\"k\">public</span> <span class=\"k\">class</span> Sample {</pre>");
+  }
+
+  @Test
+  public void useHtmlOptions() throws IOException {
+    Reader java = readFile("/org/sonar/colorizer/samples/Sample.java");
+
+    HtmlOptions options = new HtmlOptions(true, "my-table-id", false);
+    String html = CodeColorizer.javaToHtml(java, options);
+
+    assertThat(html, containsString("<table class=\"code\" id=\"my-table-id\""));
+    assertThat(html, not(containsString("<style")));
+  }
+
+  @Test
+  public void groovyToHtml() throws IOException {
+    Reader groovy = readFile("/org/sonar/colorizer/samples/Sample.groovy");
+
+    String html = CodeColorizer.groovyToHtml(groovy, HtmlOptions.DEFAULT);
+
+    assertHtml(html);
+    assertContains(html, "<pre><span class=\"k\">class</span> Greet {</pre>");
+  }
+
+  @Test
+  public void getCss() {
+    assertThat(CodeColorizer.getCss().length(), greaterThan(100));
+    assertThat(CodeColorizer.getCss(), containsString(".code"));
+  }
+
+  @Test
+  public void mustBeThreadsafe() throws FileNotFoundException, InterruptedException, ExecutionException {
+    final int taskCount = 50;
+    final int threadCount = 5;
+
+    class ColorizerTask implements Callable<String> {
+
+      Reader java;
+
+      ColorizerTask() throws FileNotFoundException {
+        this.java = readFile("/org/sonar/colorizer/samples/Sample.java");
+      }
+
+      public String call() throws Exception {
+        return CodeColorizer.javaToHtml(java, HtmlOptions.ONLY_SYNTAX);
+      }
+    }
+
+    Collection<Callable<String>> tasks = new ArrayList<Callable<String>>();
+    for (int i = 0; i < taskCount; i++) {
+      tasks.add(new ColorizerTask());
+    }
+    List<Future<String>> futures = Executors.newFixedThreadPool(threadCount).invokeAll(tasks);
+
+    assertThat(futures.size(), is(taskCount));
+
+    // all html must be the same
+    String html = futures.get(0).get();
+    for (Future<String> future : futures) {
+      assertEquals(html, future.get());
+    }
+  }
+
+  private FileReader readFile(String path) throws FileNotFoundException {
+    return new FileReader(FileUtils.toFile(getClass().getResource(path)));
+  }
+
+  private void assertHtml(String html) {
+    assertContains(html, "<style", "<table class=\"code\"", "</html>");
+  }
+
+  private void assertContains(String html, String... strings) {
+    for (String string : strings) {
+      assertThat(html, containsString(string));
+    }
+  }
+}
diff --git a/sonar-colorizer/src/test/java/org/sonar/colorizer/CppTokenizerTest.java b/sonar-colorizer/src/test/java/org/sonar/colorizer/CppTokenizerTest.java
new file mode 100644
index 00000000000..8547a69a2bc
--- /dev/null
+++ b/sonar-colorizer/src/test/java/org/sonar/colorizer/CppTokenizerTest.java
@@ -0,0 +1,59 @@
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2009 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
+ */
+package org.sonar.colorizer;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.sonar.channel.CodeReader;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class CppTokenizerTest {
+
+  private HtmlCodeBuilder codeBuilder;
+
+  @Before
+  public void init() {
+    codeBuilder = new HtmlCodeBuilder();
+  }
+
+  @Test
+  public void testRead() {
+    CppDocTokenizer javadocTokenizer = new CppDocTokenizer("<c>", "</c>");
+    assertTrue(javadocTokenizer.consume(new CodeReader("/*this is a cpp comment*/ private"), codeBuilder));
+    assertEquals("<c>/*this is a cpp comment*/</c>", codeBuilder.toString());
+    
+    assertFalse(javadocTokenizer.consume(new CodeReader("//this is not a cpp comment"), codeBuilder));
+  }
+
+  
+  @Test
+  public void testReadOnMultilines() {
+    CppDocTokenizer javadocTokenizer = new CppDocTokenizer("<c>", "</c>");
+    CodeReader reader = new CodeReader("/*this is \n a cpp comment*/ private");
+    assertTrue(javadocTokenizer.consume(reader, codeBuilder));
+    assertEquals("<c>/*this is </c>", codeBuilder.toString());
+    codeBuilder.append((char)reader.pop());
+    assertTrue(javadocTokenizer.consume(reader, codeBuilder));
+    assertEquals("<c>/*this is </c>\n<c> a cpp comment*/</c>", codeBuilder.toString());
+  }
+}
diff --git a/sonar-colorizer/src/test/java/org/sonar/colorizer/HtmlCodeBuilderTest.java b/sonar-colorizer/src/test/java/org/sonar/colorizer/HtmlCodeBuilderTest.java
new file mode 100644
index 00000000000..5c64e5e6e62
--- /dev/null
+++ b/sonar-colorizer/src/test/java/org/sonar/colorizer/HtmlCodeBuilderTest.java
@@ -0,0 +1,81 @@
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2009 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
+ */
+package org.sonar.colorizer;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.core.IsNull.nullValue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+
+public class HtmlCodeBuilderTest {
+
+  private HtmlCodeBuilder builder;
+
+  @Before
+  public void init() {
+    builder = new HtmlCodeBuilder();
+  }
+
+  @Test
+  public void testAppendCharSequence() {
+    builder.append("freddy < olivier");
+    assertEquals("freddy &lt; olivier", builder.toString());
+  }
+
+  @Test
+  public void testAppendChar() {
+    builder.append('p');
+    builder.append('a');
+    builder.append('>');
+    assertEquals("pa&gt;", builder.toString());
+  }
+
+  @Test
+  public void testAppendCharSequenceIntInt() {
+    builder.append("freddy < olivier", 0, 2);
+    assertEquals("fr", builder.toString());
+  }
+
+  @Test
+  public void testAppendWithoutTransforming() {
+    builder.appendWithoutTransforming("<inside>outside");
+    assertEquals("<inside>outside", builder.toString());
+  }
+
+  @Test
+  public void testStatefulVariables() {
+    assertThat(builder.getVariable("foo"), nullValue());
+
+    builder.setVariable("foo", "xxx");
+    assertThat((String) builder.getVariable("foo"), is("xxx"));
+
+    builder.setVariable("foo", "yyy");
+    assertThat((String) builder.getVariable("foo"), is("yyy"));
+
+    builder.setVariable("foo", null);
+    assertThat((String) builder.getVariable("foo"), nullValue());
+
+    assertThat((String) builder.getVariable("foo", "default"), is("default"));
+  }
+
+}
diff --git a/sonar-colorizer/src/test/java/org/sonar/colorizer/HtmlDecoratorTest.java b/sonar-colorizer/src/test/java/org/sonar/colorizer/HtmlDecoratorTest.java
new file mode 100644
index 00000000000..6effeaa3698
--- /dev/null
+++ b/sonar-colorizer/src/test/java/org/sonar/colorizer/HtmlDecoratorTest.java
@@ -0,0 +1,94 @@
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2009 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
+ */
+package org.sonar.colorizer;
+
+import static org.hamcrest.core.IsNot.not;
+import static org.hamcrest.number.OrderingComparisons.greaterThan;
+import static org.junit.Assert.assertThat;
+import org.junit.Test;
+import static org.junit.internal.matchers.StringContains.containsString;
+
+public class HtmlDecoratorTest {
+
+  @Test
+  public void beginOfFileTag() {
+    HtmlDecorator decorator = new HtmlDecorator(HtmlOptions.DEFAULT);
+    String tag = decorator.getTagBeginOfFile();
+    assertContains(tag, "<html", "<style", "<table");
+  }
+
+  @Test
+  public void beginOfFileTagWithTableId() {
+    HtmlOptions options = new HtmlOptions().setGenerateTable(true).setTableId("foo");
+    HtmlDecorator decorator = new HtmlDecorator(options);
+
+    String tag = decorator.getTagBeginOfFile();
+
+    assertContains(tag, "<table class=\"code\" id=\"foo\">");
+  }
+
+  @Test
+  public void beginOfFileTagWithoutHeader() {
+    HtmlOptions options = new HtmlOptions().setGenerateTable(true).setGenerateHtmlHeader(false);
+    HtmlDecorator decorator = new HtmlDecorator(options);
+
+    String tag = decorator.getTagBeginOfFile();
+
+    assertNotContains(tag, "<html", "<style");
+    assertContains(tag, "<table");
+  }
+
+  @Test
+  public void endOfFileTag() {
+    HtmlDecorator decorator = new HtmlDecorator(HtmlOptions.DEFAULT);
+    String tag = decorator.getTagEndOfFile();
+    assertContains(tag, "</table>", "</body>", "</html>");
+  }
+
+  @Test
+  public void endOfFileTagWithoutHeader() {
+    HtmlOptions options = new HtmlOptions().setGenerateTable(true).setGenerateHtmlHeader(false);
+    HtmlDecorator decorator = new HtmlDecorator(options);
+
+    String tag = decorator.getTagEndOfFile();
+
+    assertContains(tag, "</table>");
+    assertNotContains(tag, "</body>", "</html>");
+  }
+
+  @Test
+  public void getCss() {
+    assertThat(HtmlDecorator.getCss().length(), greaterThan(100));
+    assertThat(HtmlDecorator.getCss(), containsString(".code"));
+  }
+
+
+  public void assertContains(String html, String... strings) {
+    for (String string : strings) {
+      assertThat(html, containsString(string));
+    }
+  }
+
+  public void assertNotContains(String html, String... strings) {
+    for (String string : strings) {
+      assertThat(html, not(containsString(string)));
+    }
+  }
+}
diff --git a/sonar-colorizer/src/test/java/org/sonar/colorizer/HtmlRendererTest.java b/sonar-colorizer/src/test/java/org/sonar/colorizer/HtmlRendererTest.java
new file mode 100644
index 00000000000..a3c5e9b8c77
--- /dev/null
+++ b/sonar-colorizer/src/test/java/org/sonar/colorizer/HtmlRendererTest.java
@@ -0,0 +1,71 @@
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2009 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
+ */
+package org.sonar.colorizer;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.Arrays;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.Test;
+import org.sonar.channel.Channel;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+import static org.junit.internal.matchers.StringContains.containsString;
+
+public class HtmlRendererTest {
+
+  private KeywordsTokenizer javaKeywordTokenizer = new KeywordsTokenizer("<span class='k'>", "</span>", JavaKeywords.get());
+
+  @Test
+  public void renderJavaSyntax() {
+    String html = new HtmlRenderer(HtmlOptions.ONLY_SYNTAX).render(new StringReader("public class Hello {"), Arrays
+        .<Channel<HtmlCodeBuilder>> asList(javaKeywordTokenizer));
+
+    assertThat(html, is("<span class='k'>public</span> <span class='k'>class</span> Hello {"));
+  }
+
+  @Test
+  public void supportHtmlSpecialCharacters() {
+    String html = new HtmlRenderer(HtmlOptions.ONLY_SYNTAX).render(new StringReader("foo(\"<html>\");"), Arrays
+        .<Channel<HtmlCodeBuilder>> asList(new LiteralTokenizer("<s>", "</s>")));
+
+    assertThat(html, is("foo(<s>\"&lt;html&gt;\"</s>);"));
+  }
+
+  @Test
+  public void renderJavaFile() throws IOException {
+    File java = FileUtils.toFile(getClass().getResource("/org/sonar/colorizer/HtmlRendererTest/Sample.java"));
+
+    String html = new HtmlRenderer().render(new FileReader(java), Arrays.<Channel<HtmlCodeBuilder>> asList(javaKeywordTokenizer));
+
+    assertThat(html, containsString("<html>"));
+    assertThat(html, containsString("<style"));
+    assertThat(html, containsString("<table class=\"code\""));
+    assertThat(html, containsString("public"));
+    assertThat(html, containsString("class"));
+    assertThat(html, containsString("Sample"));
+    assertThat(html, containsString("</html>"));
+  }
+
+}
diff --git a/sonar-colorizer/src/test/java/org/sonar/colorizer/JavaAnnotationTokenizerTest.java b/sonar-colorizer/src/test/java/org/sonar/colorizer/JavaAnnotationTokenizerTest.java
new file mode 100644
index 00000000000..eb1fe8f0a0d
--- /dev/null
+++ b/sonar-colorizer/src/test/java/org/sonar/colorizer/JavaAnnotationTokenizerTest.java
@@ -0,0 +1,42 @@
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2009 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
+ */
+package org.sonar.colorizer;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+import static org.sonar.colorizer.SyntaxHighlighterTestingHarness.highlight;
+
+import org.junit.Test;
+
+public class JavaAnnotationTokenizerTest {
+
+  JavaAnnotationTokenizer tokenizer = new JavaAnnotationTokenizer("<a>", "</a>");
+
+  @Test
+  public void testHighlighting() {
+    assertThat(highlight("@deprecated public", tokenizer), is("<a>@deprecated</a> public"));
+    assertThat(highlight("import", tokenizer), is("import"));
+  }
+
+  @Test
+  public void testHighlightingWithProperties() {
+    assertThat(highlight("@Target(ElementType.METHOD)", tokenizer), is("<a>@Target</a>(ElementType.METHOD)"));
+  }
+}
diff --git a/sonar-colorizer/src/test/java/org/sonar/colorizer/JavaConstantTokenizerTest.java b/sonar-colorizer/src/test/java/org/sonar/colorizer/JavaConstantTokenizerTest.java
new file mode 100644
index 00000000000..0c45b78734d
--- /dev/null
+++ b/sonar-colorizer/src/test/java/org/sonar/colorizer/JavaConstantTokenizerTest.java
@@ -0,0 +1,72 @@
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2009 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
+ */
+package org.sonar.colorizer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.sonar.channel.CodeReader;
+
+public class JavaConstantTokenizerTest {
+
+  private JavaConstantTokenizer tokenizer;
+  private HtmlCodeBuilder codeBuilder;
+
+  @Before
+  public void init() {
+    tokenizer = new JavaConstantTokenizer("<c>", "</c>");
+    codeBuilder = new HtmlCodeBuilder();
+  }
+
+  @Test
+  public void testRead() {
+    assertTrue(tokenizer.consume(new CodeReader("CONSTANT"), codeBuilder));
+    assertTrue(tokenizer.consume(new CodeReader("IS_CONSTANT "), codeBuilder));
+    assertTrue(tokenizer.consume(new CodeReader("IS-CONSTANT "), codeBuilder));
+    assertFalse(tokenizer.consume(new CodeReader("Class"), codeBuilder));
+    assertFalse(tokenizer.consume(new CodeReader("_NOTACONSTANT"), codeBuilder));
+    assertFalse(tokenizer.consume(new CodeReader("IS_not_a_constant"), codeBuilder));
+    assertFalse(tokenizer.consume(new CodeReader("property"), codeBuilder));
+    assertFalse(tokenizer.consume(new CodeReader(" "), codeBuilder));
+  }
+
+  @Test
+  public void upperCaseWordsStartingWithADotShoutNotBeHighlighted() {
+    CodeReader reader = new CodeReader(".URL");
+    reader.pop();
+    assertFalse(tokenizer.consume(reader, codeBuilder));
+  }
+
+  @Test
+  public void hasNextTokenWhenFirstCharacterIsNotAConstant() {
+    CodeReader code = new CodeReader("sCONSTANT");
+    code.pop();
+    assertFalse(tokenizer.consume(code, codeBuilder));
+  }
+
+  @Test
+  public void nextToken() {
+    assertTrue(tokenizer.consume(new CodeReader("IS_TRUE = 4;"), codeBuilder));
+    assertEquals("<c>IS_TRUE</c>", codeBuilder.toString());
+  }
+}
diff --git a/sonar-colorizer/src/test/java/org/sonar/colorizer/JavaKeywordsTest.java b/sonar-colorizer/src/test/java/org/sonar/colorizer/JavaKeywordsTest.java
new file mode 100644
index 00000000000..2bd6e05484f
--- /dev/null
+++ b/sonar-colorizer/src/test/java/org/sonar/colorizer/JavaKeywordsTest.java
@@ -0,0 +1,36 @@
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2009 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
+ */
+package org.sonar.colorizer;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+import org.junit.Test;
+import static org.junit.internal.matchers.IsCollectionContaining.hasItem;
+
+public class JavaKeywordsTest {
+
+  @Test
+  public void get() {
+    assertThat(JavaKeywords.get().size(), is(53));
+    assertThat(JavaKeywords.get(), hasItem("true"));
+    assertThat(JavaKeywords.get(), hasItem("public"));
+    assertThat(JavaKeywords.get(), hasItem("switch"));
+  }
+}
diff --git a/sonar-colorizer/src/test/java/org/sonar/colorizer/JavaTokenizersTest.java b/sonar-colorizer/src/test/java/org/sonar/colorizer/JavaTokenizersTest.java
new file mode 100644
index 00000000000..dcf9e929627
--- /dev/null
+++ b/sonar-colorizer/src/test/java/org/sonar/colorizer/JavaTokenizersTest.java
@@ -0,0 +1,56 @@
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2009 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
+ */
+package org.sonar.colorizer;
+
+import java.util.List;
+
+import org.junit.Test;
+
+import static junit.framework.Assert.fail;
+
+import static org.hamcrest.number.OrderingComparisons.greaterThan;
+import static org.hamcrest.number.OrderingComparisons.lessThan;
+import static org.junit.Assert.assertThat;
+
+public class JavaTokenizersTest {
+
+  @Test
+  public void forHtml() {
+    assertThat(JavaTokenizers.forHtml().size(), greaterThan(3));
+  }
+
+  @Test
+  public void javadocIsDefinedBeforeCppComment() {
+    // just because /** must be detected before /*
+    assertThat(indexOf(JavaTokenizers.forHtml(), JavadocTokenizer.class),
+        lessThan(indexOf(JavaTokenizers.forHtml(), CppDocTokenizer.class)));
+  }
+
+  private Integer indexOf(List<Tokenizer> tokenizers, Class tokenizerClass) {
+    for (int i = 0; i < tokenizers.size(); i++) {
+      if (tokenizers.get(i).getClass().equals(tokenizerClass)) {
+        return i;
+      }
+    }
+
+    fail("Tokenizer not found: " + tokenizerClass);
+    return null;
+  }
+}
diff --git a/sonar-colorizer/src/test/java/org/sonar/colorizer/JavadocTokenizerTest.java b/sonar-colorizer/src/test/java/org/sonar/colorizer/JavadocTokenizerTest.java
new file mode 100644
index 00000000000..08a560e0585
--- /dev/null
+++ b/sonar-colorizer/src/test/java/org/sonar/colorizer/JavadocTokenizerTest.java
@@ -0,0 +1,42 @@
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2009 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
+ */
+package org.sonar.colorizer;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+import static org.sonar.colorizer.SyntaxHighlighterTestingHarness.highlight;
+
+import org.junit.Test;
+
+public class JavadocTokenizerTest {
+
+  JavadocTokenizer tokenizer = new JavadocTokenizer("<j>", "</j>");
+
+  @Test
+  public void testHighlighting() {
+    assertThat(highlight("/**this is a javadoc*/ public ...", tokenizer), is("<j>/**this is a javadoc*/</j> public ..."));
+    assertThat(highlight("//this is not a javadoc", tokenizer), is("//this is not a javadoc"));
+  }
+
+  @Test
+  public void testHighlightingOnMultipleLines() {
+    assertThat(highlight("/**this is \n a javadoc*/ private", tokenizer), is("<j>/**this is </j>\n<j> a javadoc*/</j> private"));
+  }
+}
diff --git a/sonar-colorizer/src/test/java/org/sonar/colorizer/KeywordsTokenizerTest.java b/sonar-colorizer/src/test/java/org/sonar/colorizer/KeywordsTokenizerTest.java
new file mode 100644
index 00000000000..5a5968f59d8
--- /dev/null
+++ b/sonar-colorizer/src/test/java/org/sonar/colorizer/KeywordsTokenizerTest.java
@@ -0,0 +1,58 @@
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2009 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
+ */
+package org.sonar.colorizer;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.Matchers.not;
+import static org.junit.Assert.assertThat;
+import static org.sonar.colorizer.SyntaxHighlighterTestingHarness.highlight;
+
+import org.junit.Test;
+
+public class KeywordsTokenizerTest {
+
+  @Test
+  public void testColorizeKeywords() {
+    KeywordsTokenizer tokenizer = new KeywordsTokenizer("<s>", "</s>", "public", "new");
+    assertThat(highlight("new()", tokenizer), is("<s>new</s>()"));
+    assertThat(highlight("public new get()", tokenizer), is("<s>public</s> <s>new</s> get()"));
+    assertThat(highlight("publication", tokenizer), is("publication"));
+  }
+
+  @Test
+  public void testUnderscoreAndDigit() {
+    KeywordsTokenizer tokenizer = new KeywordsTokenizer("<s>", "</s>", "_01public");
+    assertThat(highlight("_01public", tokenizer), is("<s>_01public</s>"));
+  }
+
+  @Test
+  public void testCaseSensitive() {
+    KeywordsTokenizer tokenizer = new KeywordsTokenizer("<s>", "</s>", "public");
+    assertThat(highlight("PUBLIC Public public", tokenizer), is("PUBLIC Public <s>public</s>"));
+  }
+  
+  @Test
+  public void testClone() {
+    KeywordsTokenizer tokenizer = new KeywordsTokenizer("<s>", "</s>", "public", "[a-z]+");
+    KeywordsTokenizer cloneTokenizer = tokenizer.clone();
+    assertThat(tokenizer, is(not(cloneTokenizer)));
+    assertThat(highlight("public 1234", cloneTokenizer), is("<s>public</s> 1234"));
+  }
+}
diff --git a/sonar-colorizer/src/test/java/org/sonar/colorizer/LiteralTokenizerTest.java b/sonar-colorizer/src/test/java/org/sonar/colorizer/LiteralTokenizerTest.java
new file mode 100644
index 00000000000..cb975763cc2
--- /dev/null
+++ b/sonar-colorizer/src/test/java/org/sonar/colorizer/LiteralTokenizerTest.java
@@ -0,0 +1,59 @@
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2009 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
+ */
+package org.sonar.colorizer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.sonar.channel.CodeReader;
+
+public class LiteralTokenizerTest {
+
+  private HtmlCodeBuilder codeBuilder;
+
+  @Before
+  public void init() {
+    codeBuilder = new HtmlCodeBuilder();
+  }
+
+  @Test
+  public void nextToken() {
+    LiteralTokenizer tokenizer = new LiteralTokenizer("<s>", "</s>");
+    assertTrue(tokenizer.consume(new CodeReader("\"toto\";"), codeBuilder));
+    assertEquals("<s>\"toto\"</s>", codeBuilder.toString());
+    init();
+    assertTrue(tokenizer.consume(new CodeReader("\"\";"), codeBuilder));
+    assertEquals("<s>\"\"</s>", codeBuilder.toString());
+    init();
+    assertTrue(tokenizer.consume(new CodeReader("\'\';"), codeBuilder));
+    assertEquals("<s>\'\'</s>", codeBuilder.toString());
+    init();
+    assertTrue(tokenizer.consume(new CodeReader("\"(\\\"\").replace"), codeBuilder));
+    assertEquals("<s>\"(\\\"\"</s>", codeBuilder.toString());
+    init();
+    assertTrue(tokenizer.consume(new CodeReader("\'\\\\\';"), codeBuilder));
+    assertEquals("<s>\'\\\\\'</s>", codeBuilder.toString());
+    init();
+    assertTrue(tokenizer.consume(new CodeReader("\"to\\'to\""), codeBuilder));
+    assertEquals("<s>\"to\\'to\"</s>", codeBuilder.toString());
+  }
+}
diff --git a/sonar-colorizer/src/test/java/org/sonar/colorizer/MultilinesDocTokenizerTest.java b/sonar-colorizer/src/test/java/org/sonar/colorizer/MultilinesDocTokenizerTest.java
new file mode 100644
index 00000000000..32a6f65d1d6
--- /dev/null
+++ b/sonar-colorizer/src/test/java/org/sonar/colorizer/MultilinesDocTokenizerTest.java
@@ -0,0 +1,80 @@
+/*

+ * Sonar, open source software quality management tool.

+ * Copyright (C) 2009 SonarSource SA

+ * mailto:contact AT sonarsource DOT com

+ *

+ * Sonar is free software; you can redistribute it and/or

+ * modify it under the terms of the GNU Lesser General Public

+ * License as published by the Free Software Foundation; either

+ * version 3 of the License, or (at your option) any later version.

+ *

+ * Sonar is distributed in the hope that it will be useful,

+ * but WITHOUT ANY WARRANTY; without even the implied warranty of

+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

+ * Lesser General Public License for more details.

+ *

+ * You should have received a copy of the GNU Lesser General Public

+ * License along with Sonar; if not, write to the Free Software

+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02

+ */

+

+package org.sonar.colorizer;

+

+import org.junit.Before;

+import org.junit.Test;

+import org.sonar.channel.CodeReader;

+

+import static org.hamcrest.MatcherAssert.assertThat;

+import static org.hamcrest.Matchers.is;

+import static org.junit.Assert.assertEquals;

+import static org.junit.Assert.assertTrue;

+

+public class MultilinesDocTokenizerTest {

+

+  private HtmlCodeBuilder codeBuilder;

+

+  @Before

+  public void init() {

+    codeBuilder = new HtmlCodeBuilder();

+  }

+

+  @Test

+  public void testStandardComment() {

+    MultilinesDocTokenizer tokenizer = new MultiLineDocTokenizerImpl("{[||", "");

+    assertThat(tokenizer.hasNextToken(new CodeReader("{[|| And here is strange  multi-line comment"), new HtmlCodeBuilder()), is(true));

+    assertThat(tokenizer.hasNextToken(new CodeReader("// this is not a strange multi-line comment"), new HtmlCodeBuilder()), is(false));

+  }

+

+  @Test

+  public void testLongStartToken() {

+    MultilinesDocTokenizer tokenizer = new MultiLineDocTokenizerImpl("/***", "**/");

+    assertTrue(tokenizer.consume(new CodeReader("/*** multi-line comment**/ private part"), codeBuilder));

+    assertEquals("/*** multi-line comment**/", codeBuilder.toString());

+  }

+  

+  @Test

+  public void testStartTokenEndTokenOverlapping() {

+    MultilinesDocTokenizer tokenizer = new MultiLineDocTokenizerImpl("/*", "*/");

+    assertTrue(tokenizer.consume(new CodeReader("/*// multi-line comment*/ private part"), codeBuilder));

+    assertEquals("/*// multi-line comment*/", codeBuilder.toString());

+  }

+  

+  @Test

+  public void testMultilinesComment() {

+    CodeReader reader = new CodeReader("/* multi-line comment\n*/ private part");

+    MultilinesDocTokenizer tokenizer = new MultiLineDocTokenizerImpl("/*", "*/");

+    assertTrue(tokenizer.consume(reader, codeBuilder));

+    assertEquals("/* multi-line comment", codeBuilder.toString());

+    reader.pop();

+    assertTrue(tokenizer.consume(reader, codeBuilder));

+    assertEquals("/* multi-line comment*/", codeBuilder.toString());

+  }

+

+  public class MultiLineDocTokenizerImpl extends MultilinesDocTokenizer {

+

+    public MultiLineDocTokenizerImpl(String startToken, String endToken) {

+      super(startToken, endToken, "", "");

+    }

+  }

+

+}

diff --git a/sonar-colorizer/src/test/java/org/sonar/colorizer/RegexpTokenizerTest.java b/sonar-colorizer/src/test/java/org/sonar/colorizer/RegexpTokenizerTest.java
new file mode 100644
index 00000000000..eb400e14a13
--- /dev/null
+++ b/sonar-colorizer/src/test/java/org/sonar/colorizer/RegexpTokenizerTest.java
@@ -0,0 +1,28 @@
+package org.sonar.colorizer;
+
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+import static org.junit.Assert.assertThat;
+import static org.sonar.colorizer.SyntaxHighlighterTestingHarness.highlight;
+
+import org.junit.Test;
+
+public class RegexpTokenizerTest {
+
+  RegexpTokenizer tokenHighlighter;;
+
+  @Test
+  public void testHighlight() {
+    tokenHighlighter = new RegexpTokenizer("<r>", "</r>", "[0-9]+");
+    assertThat(highlight("123, word = 435;", tokenHighlighter), is("<r>123</r>, word = <r>435</r>;"));
+  }
+  
+  @Test
+  public void testClone() {
+    RegexpTokenizer tokenizer = new RegexpTokenizer("<r>", "</r>", "[a-z]+");
+    RegexpTokenizer cloneTokenizer = tokenizer.clone();
+    assertThat(tokenizer, is(not(cloneTokenizer)));
+    assertThat(highlight("public 1234", cloneTokenizer), is("<r>public</r> 1234"));
+  }
+
+}
diff --git a/sonar-colorizer/src/test/java/org/sonar/colorizer/SyntaxHighlighterTestingHarness.java b/sonar-colorizer/src/test/java/org/sonar/colorizer/SyntaxHighlighterTestingHarness.java
new file mode 100644
index 00000000000..882752e8bb9
--- /dev/null
+++ b/sonar-colorizer/src/test/java/org/sonar/colorizer/SyntaxHighlighterTestingHarness.java
@@ -0,0 +1,11 @@
+package org.sonar.colorizer;
+
+import org.sonar.channel.Channel;
+
+public class SyntaxHighlighterTestingHarness {
+
+  public static String highlight(String input, Channel<HtmlCodeBuilder> tokenHighlighter) {
+    TokenizerDispatcher syntaxHighlighter = new TokenizerDispatcher(tokenHighlighter);
+    return syntaxHighlighter.colorize(input);
+  }
+}
diff --git a/sonar-colorizer/src/test/java/org/sonar/colorizer/TokenizerDispatcherTest.java b/sonar-colorizer/src/test/java/org/sonar/colorizer/TokenizerDispatcherTest.java
new file mode 100644
index 00000000000..c551b289e03
--- /dev/null
+++ b/sonar-colorizer/src/test/java/org/sonar/colorizer/TokenizerDispatcherTest.java
@@ -0,0 +1,81 @@
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2009 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
+ */
+package org.sonar.colorizer;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+import java.util.Arrays;
+
+import org.junit.Test;
+import org.sonar.channel.Channel;
+import org.sonar.channel.CodeReader;
+
+public class TokenizerDispatcherTest {
+
+  @Test
+  public void testPipeCodeTokenizer() {
+    TokenizerDispatcher colorization = newColorizer();
+    assertThat(colorization.colorize("public void get(){"), is("public void get(){"));
+  }
+
+  @Test
+  public void testKeywordsCodeTokenizer() {
+    TokenizerDispatcher colorization = newColorizer(new KeywordsTokenizer("<k>", "</k>", JavaKeywords.get()));
+    assertThat(colorization.colorize("public void get(){"), is("<k>public</k> <k>void</k> get(){"));
+  }
+
+  @Test
+  public void testPriorityToComment() {
+    TokenizerDispatcher colorization = newColorizer(new CDocTokenizer("<c>", "</c>"), new KeywordsTokenizer("<k>", "</k>", JavaKeywords
+        .get()));
+    assertThat(colorization.colorize("assert //public void get(){"), is("<k>assert</k> <c>//public void get(){</c>"));
+  }
+
+  @Test
+  public void testCommentThenStringThenJavaKeywords() {
+    TokenizerDispatcher colorization = newColorizer(new CDocTokenizer("<c>", "</c>"), new LiteralTokenizer("<s>", "</s>"),
+        new KeywordsTokenizer("<k>", "</k>", JavaKeywords.get()));
+    assertThat(colorization.colorize("assert(\"message\"); //comment"), is("<k>assert</k>(<s>\"message\"</s>); <c>//comment</c>"));
+  }
+
+  @Test(expected = IllegalStateException.class)
+  public void testCloneNotThreadSafeTokenizers() {
+    NotThreadSafeTokenizer tokenizer = new NotThreadSafeTokenizer() {
+
+      @Override
+      public boolean consume(CodeReader code, HtmlCodeBuilder output) {
+        output.append((char) code.pop());
+        return true;
+      }
+
+      @Override
+      public NotThreadSafeTokenizer clone() {
+        throw new IllegalStateException("The clone method has been called as expected.");
+      }
+    };
+    TokenizerDispatcher colorization = newColorizer(tokenizer);
+    colorization.colorize("source code");
+  }
+
+  private TokenizerDispatcher newColorizer(Channel<HtmlCodeBuilder>... tokenizers) {
+    return new TokenizerDispatcher(Arrays.asList(tokenizers));
+  }
+}
diff --git a/sonar-colorizer/src/test/java/org/sonar/colorizer/UserGuideTest.java b/sonar-colorizer/src/test/java/org/sonar/colorizer/UserGuideTest.java
new file mode 100644
index 00000000000..681e810c127
--- /dev/null
+++ b/sonar-colorizer/src/test/java/org/sonar/colorizer/UserGuideTest.java
@@ -0,0 +1,76 @@
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2009 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02
+ */
+package org.sonar.colorizer;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.Reader;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.Test;
+
+public class UserGuideTest {
+
+  @Test
+  public void javaToHtml() throws IOException {
+    Reader java = readFile("/org/sonar/colorizer/samples/Sample.java");
+
+    String html = CodeColorizer.javaToHtml(java, HtmlOptions.DEFAULT);
+
+    save(html, "Sample.java.html");
+  }
+
+  @Test
+  public void groovyToHtml() throws IOException {
+    Reader groovy = readFile("/org/sonar/colorizer/samples/Sample.groovy");
+
+    String html = CodeColorizer.groovyToHtml(groovy, HtmlOptions.DEFAULT);
+
+    save(html, "Sample.groovy.html");
+  }
+
+  @Test
+  public void customizeOutput() throws IOException {
+    Reader java = readFile("/org/sonar/colorizer/samples/Sample.java");
+
+    // generate only the <table> element, without including the CSS and the HTML header
+    HtmlOptions options = new HtmlOptions(true, "my-table-id", false);
+    String html = CodeColorizer.javaToHtml(java, options);
+
+    save(html, "CustomizeOutput.java.html");
+  }
+
+  @Test
+  public void defineNewLanguage() throws IOException {
+
+  }
+
+  private FileReader readFile(String path) throws FileNotFoundException {
+    return new FileReader(FileUtils.toFile(getClass().getResource(path)));
+  }
+
+  private void save(String html, String filename) throws IOException {
+    File output = new File("target/userguide/" + filename);
+    FileUtils.writeStringToFile(output, html);
+    System.out.println("HTML sample saved to: " + output.getAbsolutePath());
+  }
+}
diff --git a/sonar-colorizer/src/test/resources/org/sonar/colorizer/HtmlRendererTest/Sample.java b/sonar-colorizer/src/test/resources/org/sonar/colorizer/HtmlRendererTest/Sample.java
new file mode 100644
index 00000000000..6f8b935e1ae
--- /dev/null
+++ b/sonar-colorizer/src/test/resources/org/sonar/colorizer/HtmlRendererTest/Sample.java
@@ -0,0 +1,17 @@
+package org.sonar.colorizer.colorizer.HtmlRendererTest;
+
+public class Sample {
+
+  /**
+   * some javadoc
+   *
+   * @return foo
+   */
+  @Deprecated
+  public String foo(String bar) {
+    if (true) {
+      return "foo";
+    }
+    return "bar";
+  }
+}
\ No newline at end of file
diff --git a/sonar-colorizer/src/test/resources/org/sonar/colorizer/samples/Sample.groovy b/sonar-colorizer/src/test/resources/org/sonar/colorizer/samples/Sample.groovy
new file mode 100644
index 00000000000..70de12a2e30
--- /dev/null
+++ b/sonar-colorizer/src/test/resources/org/sonar/colorizer/samples/Sample.groovy
@@ -0,0 +1,13 @@
+/**
+ comment
+ */
+@Deprecated
+class Greet {
+  def name
+  Greet(who) { name = who[0].toUpperCase() +
+                      who[1..-1] }
+  def salute() { println "Hello $name!" }
+}
+
+g = new Greet('world')  // create object
+g.salute()              // Output "Hello World!"
diff --git a/sonar-colorizer/src/test/resources/org/sonar/colorizer/samples/Sample.java b/sonar-colorizer/src/test/resources/org/sonar/colorizer/samples/Sample.java
new file mode 100644
index 00000000000..f736b368147
--- /dev/null
+++ b/sonar-colorizer/src/test/resources/org/sonar/colorizer/samples/Sample.java
@@ -0,0 +1,52 @@
+package org.sonar.colorizer.samples;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Target;
+import java.net.URL;
+
+/*
+ * NOT javadoc, for example license header
+*/
+public class Sample {
+
+  private static final double NUMBER = 3.1415;
+  private static final double INTEGER = 77;
+
+  /**
+   * some javadoc
+   *
+   * @return foo
+   */
+  @Deprecated
+  public String foo(String bar) {
+    // single-line comment
+    if (true) {
+      return "foo";
+    }
+    
+    String backslashes = "";
+    backslashes.replaceAll("\\(", "(\"").replaceAll("\\)", "\")");
+
+    /*
+      Multi-lines comment
+     */
+    return "bar";
+  }
+
+  /**
+   * Special HTML characters in code : <code>foo</code>
+   */
+  public void specialHtmlCharacters() {
+    String special = "<html> <<";
+  }
+
+
+  public native void nativeMethod() /*-{
+    Window.alert('this is javascript');
+  }-*/;
+}
+
+@Target(ElementType.METHOD)
+    @interface Foo {
+
+}
\ No newline at end of file
diff --git a/sonar-colorizer/src/test/resources/org/sonar/colorizer/samples/SingleLine.java b/sonar-colorizer/src/test/resources/org/sonar/colorizer/samples/SingleLine.java
new file mode 100644
index 00000000000..a9d082df1e6
--- /dev/null
+++ b/sonar-colorizer/src/test/resources/org/sonar/colorizer/samples/SingleLine.java
@@ -0,0 +1 @@
+package org.sonar.colorizer.samples;
\ No newline at end of file