]> source.dussan.org Git - sonarqube.git/commitdiff
SONAR-5827 Execute CodeColorizer on batch side to feed highlighting
authorJulien HENRY <julien.henry@sonarsource.com>
Fri, 21 Nov 2014 15:01:44 +0000 (16:01 +0100)
committerJulien HENRY <julien.henry@sonarsource.com>
Fri, 21 Nov 2014 15:17:02 +0000 (16:17 +0100)
16 files changed:
plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/CorePlugin.java
plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/colorizers/JavaColorizerFormat.java [deleted file]
plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/colorizers/package-info.java [deleted file]
sonar-batch/src/main/java/org/sonar/batch/highlighting/SyntaxHighlightingData.java
sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java
sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java
sonar-batch/src/main/java/org/sonar/batch/source/CodeColorizers.java [new file with mode: 0644]
sonar-batch/src/main/java/org/sonar/batch/source/HighlightingCodeBuilder.java [new file with mode: 0644]
sonar-batch/src/main/java/org/sonar/batch/source/HighlightingRenderer.java [new file with mode: 0644]
sonar-batch/src/test/java/org/sonar/batch/highlighting/DefaultHighlightingBuilderTest.java
sonar-batch/src/test/java/org/sonar/batch/highlighting/SyntaxHighlightingDataTest.java
sonar-batch/src/test/java/org/sonar/batch/index/SourcePersisterTest.java
sonar-batch/src/test/java/org/sonar/batch/source/CodeColorizersTest.java [new file with mode: 0644]
sonar-batch/src/test/java/org/sonar/batch/source/DefaultHighlightableTest.java
sonar-batch/src/test/resources/org/sonar/batch/source/CodeColorizersTest/Person.js [new file with mode: 0644]
sonar-plugin-api/src/main/java/org/sonar/api/web/CodeColorizerFormat.java

index 0fddfec945613132fa85a38e20d0fa50a6a77ecb..9b996f161617cedc5c9ec2276e02e4bef683f44e 100644 (file)
 package org.sonar.plugins.core;
 
 import com.google.common.collect.ImmutableList;
-import org.sonar.api.*;
+import org.sonar.api.CoreProperties;
+import org.sonar.api.Properties;
+import org.sonar.api.Property;
+import org.sonar.api.PropertyType;
+import org.sonar.api.SonarPlugin;
 import org.sonar.api.checks.NoSonarFilter;
 import org.sonar.core.timemachine.Periods;
 import org.sonar.plugins.core.charts.DistributionAreaChart;
 import org.sonar.plugins.core.charts.DistributionBarChart;
-import org.sonar.plugins.core.colorizers.JavaColorizerFormat;
 import org.sonar.plugins.core.dashboards.GlobalDefaultDashboard;
 import org.sonar.plugins.core.dashboards.ProjectDefaultDashboard;
 import org.sonar.plugins.core.dashboards.ProjectIssuesDashboard;
 import org.sonar.plugins.core.dashboards.ProjectTimeMachineDashboard;
-import org.sonar.plugins.core.issue.*;
-import org.sonar.plugins.core.issue.notification.*;
+import org.sonar.plugins.core.issue.CountFalsePositivesDecorator;
+import org.sonar.plugins.core.issue.CountUnresolvedIssuesDecorator;
+import org.sonar.plugins.core.issue.InitialOpenIssuesSensor;
+import org.sonar.plugins.core.issue.InitialOpenIssuesStack;
+import org.sonar.plugins.core.issue.IssueHandlers;
+import org.sonar.plugins.core.issue.IssueTracking;
+import org.sonar.plugins.core.issue.IssueTrackingDecorator;
+import org.sonar.plugins.core.issue.notification.ChangesOnMyIssueNotificationDispatcher;
+import org.sonar.plugins.core.issue.notification.IssueChangesEmailTemplate;
+import org.sonar.plugins.core.issue.notification.NewFalsePositiveNotificationDispatcher;
+import org.sonar.plugins.core.issue.notification.NewIssuesEmailTemplate;
+import org.sonar.plugins.core.issue.notification.NewIssuesNotificationDispatcher;
+import org.sonar.plugins.core.issue.notification.SendIssueNotificationsPostJob;
 import org.sonar.plugins.core.measurefilters.MyFavouritesFilter;
 import org.sonar.plugins.core.measurefilters.ProjectFilter;
 import org.sonar.plugins.core.notifications.alerts.NewAlerts;
 import org.sonar.plugins.core.security.ApplyProjectRolesDecorator;
-import org.sonar.plugins.core.sensors.*;
-import org.sonar.plugins.core.timemachine.*;
-import org.sonar.plugins.core.widgets.*;
-import org.sonar.plugins.core.widgets.issues.*;
-import org.sonar.plugins.core.widgets.measures.*;
+import org.sonar.plugins.core.sensors.BranchCoverageDecorator;
+import org.sonar.plugins.core.sensors.CommentDensityDecorator;
+import org.sonar.plugins.core.sensors.CoverageDecorator;
+import org.sonar.plugins.core.sensors.CoverageMeasurementFilter;
+import org.sonar.plugins.core.sensors.DirectoriesDecorator;
+import org.sonar.plugins.core.sensors.FileHashSensor;
+import org.sonar.plugins.core.sensors.FilesDecorator;
+import org.sonar.plugins.core.sensors.ItBranchCoverageDecorator;
+import org.sonar.plugins.core.sensors.ItCoverageDecorator;
+import org.sonar.plugins.core.sensors.ItLineCoverageDecorator;
+import org.sonar.plugins.core.sensors.LineCoverageDecorator;
+import org.sonar.plugins.core.sensors.ManualMeasureDecorator;
+import org.sonar.plugins.core.sensors.OverallBranchCoverageDecorator;
+import org.sonar.plugins.core.sensors.OverallCoverageDecorator;
+import org.sonar.plugins.core.sensors.OverallLineCoverageDecorator;
+import org.sonar.plugins.core.sensors.ProjectLinksSensor;
+import org.sonar.plugins.core.sensors.UnitTestDecorator;
+import org.sonar.plugins.core.sensors.VersionEventsSensor;
+import org.sonar.plugins.core.timemachine.NewCoverageAggregator;
+import org.sonar.plugins.core.timemachine.NewCoverageFileAnalyzer;
+import org.sonar.plugins.core.timemachine.NewItCoverageFileAnalyzer;
+import org.sonar.plugins.core.timemachine.NewOverallCoverageFileAnalyzer;
+import org.sonar.plugins.core.timemachine.TendencyDecorator;
+import org.sonar.plugins.core.timemachine.TimeMachineConfigurationPersister;
+import org.sonar.plugins.core.timemachine.VariationDecorator;
+import org.sonar.plugins.core.widgets.AlertsWidget;
+import org.sonar.plugins.core.widgets.BubbleChartWidget;
+import org.sonar.plugins.core.widgets.ComplexityWidget;
+import org.sonar.plugins.core.widgets.CoverageWidget;
+import org.sonar.plugins.core.widgets.CustomMeasuresWidget;
+import org.sonar.plugins.core.widgets.DebtOverviewWidget;
+import org.sonar.plugins.core.widgets.DescriptionWidget;
+import org.sonar.plugins.core.widgets.DocumentationCommentsWidget;
+import org.sonar.plugins.core.widgets.DuplicationsWidget;
+import org.sonar.plugins.core.widgets.EventsWidget;
+import org.sonar.plugins.core.widgets.HotspotMetricWidget;
+import org.sonar.plugins.core.widgets.HotspotMostViolatedRulesWidget;
+import org.sonar.plugins.core.widgets.ItCoverageWidget;
+import org.sonar.plugins.core.widgets.ProjectFileCloudWidget;
+import org.sonar.plugins.core.widgets.SizeWidget;
+import org.sonar.plugins.core.widgets.TechnicalDebtPyramidWidget;
+import org.sonar.plugins.core.widgets.TimeMachineWidget;
+import org.sonar.plugins.core.widgets.TimelineWidget;
+import org.sonar.plugins.core.widgets.TreemapWidget;
+import org.sonar.plugins.core.widgets.WelcomeWidget;
+import org.sonar.plugins.core.widgets.issues.ActionPlansWidget;
+import org.sonar.plugins.core.widgets.issues.FalsePositiveIssuesWidget;
+import org.sonar.plugins.core.widgets.issues.IssueFilterWidget;
+import org.sonar.plugins.core.widgets.issues.IssuesWidget;
+import org.sonar.plugins.core.widgets.issues.MyUnresolvedIssuesWidget;
+import org.sonar.plugins.core.widgets.issues.UnresolvedIssuesPerAssigneeWidget;
+import org.sonar.plugins.core.widgets.issues.UnresolvedIssuesStatusesWidget;
+import org.sonar.plugins.core.widgets.measures.MeasureFilterAsBubbleChartWidget;
+import org.sonar.plugins.core.widgets.measures.MeasureFilterAsCloudWidget;
+import org.sonar.plugins.core.widgets.measures.MeasureFilterAsHistogramWidget;
+import org.sonar.plugins.core.widgets.measures.MeasureFilterAsPieChartWidget;
+import org.sonar.plugins.core.widgets.measures.MeasureFilterAsTreemapWidget;
+import org.sonar.plugins.core.widgets.measures.MeasureFilterListWidget;
 
 import java.util.List;
 
@@ -261,9 +328,6 @@ public final class CorePlugin extends SonarPlugin {
       DistributionBarChart.class,
       DistributionAreaChart.class,
 
-      // colorizers
-      JavaColorizerFormat.class,
-
       // issues
       IssueTrackingDecorator.class,
       IssueTracking.class,
diff --git a/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/colorizers/JavaColorizerFormat.java b/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/colorizers/JavaColorizerFormat.java
deleted file mode 100644 (file)
index 09f43cd..0000000
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.plugins.core.colorizers;
-
-import org.sonar.api.web.CodeColorizerFormat;
-import org.sonar.colorizer.CodeColorizer;
-import org.sonar.colorizer.Tokenizer;
-
-import java.util.List;
-
-public class JavaColorizerFormat extends CodeColorizerFormat{
-
-  public JavaColorizerFormat() {
-    super("java");
-  }
-
-  @Override
-  public List<Tokenizer> getTokenizers() {
-    return CodeColorizer.Format.JAVA.getTokenizers();
-  }
-}
diff --git a/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/colorizers/package-info.java b/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/colorizers/package-info.java
deleted file mode 100644 (file)
index bd332ae..0000000
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-/**
- * Deprecated in 4.5.1. JFreechart charts are replaced by Javascript charts.
- */
-@ParametersAreNonnullByDefault
-package org.sonar.plugins.core.colorizers;
-
-import javax.annotation.ParametersAreNonnullByDefault;
index 65c1633a039d61bf68805ba2cc014b41d27675e5..b1075a9ed192327887e3ab2fdae7a7b496bc663c 100644 (file)
@@ -44,12 +44,14 @@ public class SyntaxHighlightingData implements Data {
   public String writeString() {
     StringBuilder sb = new StringBuilder();
     for (SyntaxHighlightingRule highlightingRule : syntaxHighlightingRuleSet) {
+      if (sb.length() > 0) {
+        sb.append(RULE_SEPARATOR);
+      }
       sb.append(highlightingRule.getStartPosition())
         .append(FIELD_SEPARATOR)
         .append(highlightingRule.getEndPosition())
         .append(FIELD_SEPARATOR)
-        .append(highlightingRule.getTextType().cssClass())
-        .append(RULE_SEPARATOR);
+        .append(highlightingRule.getTextType().cssClass());
     }
 
     return sb.toString();
index 0ff6fbe57c2d9eec2ef1e19a1babfc36df4e3369..0ce9d1f7737c8f39dbe7a266bd85c14bf9d26e45 100644 (file)
@@ -41,6 +41,7 @@ import org.sonar.batch.highlighting.SyntaxHighlightingData;
 import org.sonar.batch.highlighting.SyntaxHighlightingRule;
 import org.sonar.batch.scan.filesystem.InputPathCache;
 import org.sonar.batch.scan.measure.MeasureCache;
+import org.sonar.batch.source.CodeColorizers;
 import org.sonar.core.persistence.DbSession;
 import org.sonar.core.persistence.MyBatis;
 import org.sonar.core.source.SnapshotDataTypes;
@@ -76,9 +77,11 @@ public class SourcePersister implements ScanPersister {
   private final System2 system2;
   private final ProjectTree projectTree;
   private final ResourceCache resourceCache;
+  private CodeColorizers codeColorizers;
 
   public SourcePersister(ResourcePersister resourcePersister, SnapshotSourceDao sourceDao, InputPathCache inputPathCache,
-    MyBatis mybatis, MeasureCache measureCache, ComponentDataCache componentDataCache, ProjectTree projectTree, System2 system2, ResourceCache resourceCache) {
+    MyBatis mybatis, MeasureCache measureCache, ComponentDataCache componentDataCache, ProjectTree projectTree, System2 system2, ResourceCache resourceCache,
+    CodeColorizers codeColorizers) {
     this.resourcePersister = resourcePersister;
     this.sourceDao = sourceDao;
     this.inputPathCache = inputPathCache;
@@ -88,6 +91,7 @@ public class SourcePersister implements ScanPersister {
     this.projectTree = projectTree;
     this.system2 = system2;
     this.resourceCache = resourceCache;
+    this.codeColorizers = codeColorizers;
   }
 
   public void saveSource(Resource resource, String source, Date updatedAt) {
@@ -180,7 +184,7 @@ public class SourcePersister implements ScanPersister {
     Map<Integer, String> authorsByLine = getLineMetric(file, CoreMetrics.SCM_AUTHORS_BY_LINE_KEY);
     Map<Integer, String> revisionsByLine = getLineMetric(file, CoreMetrics.SCM_REVISIONS_BY_LINE_KEY);
     Map<Integer, String> datesByLine = getLineMetric(file, CoreMetrics.SCM_LAST_COMMIT_DATETIMES_BY_LINE_KEY);
-    SyntaxHighlightingData highlighting = componentDataCache.getData(file.key(), SnapshotDataTypes.SYNTAX_HIGHLIGHTING);
+    SyntaxHighlightingData highlighting = loadHighlighting(file);
     String[] highlightingPerLine = computeHighlightingPerLine(file, highlighting);
 
     ByteArrayOutputStream output = new ByteArrayOutputStream();
@@ -193,6 +197,15 @@ public class SourcePersister implements ScanPersister {
     return StringUtils.defaultIfEmpty(new String(output.toByteArray(), UTF_8), null);
   }
 
+  @CheckForNull
+  private SyntaxHighlightingData loadHighlighting(DefaultInputFile file) {
+    SyntaxHighlightingData highlighting = componentDataCache.getData(file.key(), SnapshotDataTypes.SYNTAX_HIGHLIGHTING);
+    if (highlighting == null) {
+      highlighting = codeColorizers.toSyntaxHighlighting(file.file(), file.encoding(), file.language());
+    }
+    return highlighting;
+  }
+
   String[] computeHighlightingPerLine(DefaultInputFile file, @Nullable SyntaxHighlightingData highlighting) {
     String[] result = new String[file.lines()];
     if (highlighting == null) {
index 3c881aada8842c85e1ac3dddce0e66a530afb007..e1a1467a95f447c947019faa2ae1968d1b6a616a 100644 (file)
@@ -73,6 +73,7 @@ import org.sonar.batch.scan.filesystem.InputPathCache;
 import org.sonar.batch.scan.maven.FakeMavenPluginExecutor;
 import org.sonar.batch.scan.maven.MavenPluginExecutor;
 import org.sonar.batch.scan.measure.MeasureCache;
+import org.sonar.batch.source.CodeColorizers;
 import org.sonar.batch.source.HighlightableBuilder;
 import org.sonar.batch.source.SymbolizableBuilder;
 import org.sonar.core.component.ScanGraph;
@@ -140,6 +141,7 @@ public class ProjectScanContainer extends ComponentContainer {
       DuplicationPersister.class,
       DefaultResourcePersister.class,
       SourcePersister.class,
+      CodeColorizers.class,
       DefaultNotificationManager.class,
       MetricProvider.class,
       ProjectConfigurator.class,
diff --git a/sonar-batch/src/main/java/org/sonar/batch/source/CodeColorizers.java b/sonar-batch/src/main/java/org/sonar/batch/source/CodeColorizers.java
new file mode 100644 (file)
index 0000000..a1d3ce9
--- /dev/null
@@ -0,0 +1,84 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.batch.source;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonar.api.BatchComponent;
+import org.sonar.api.web.CodeColorizerFormat;
+import org.sonar.batch.highlighting.SyntaxHighlightingData;
+import org.sonar.colorizer.Tokenizer;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Central point for sonar-colorizer extensions
+ */
+public class CodeColorizers implements BatchComponent {
+
+  private static final Logger LOG = LoggerFactory.getLogger(CodeColorizers.class);
+
+  private final Map<String, CodeColorizerFormat> byLang;
+
+  public CodeColorizers(List<CodeColorizerFormat> formats) {
+    byLang = new HashMap<String, CodeColorizerFormat>();
+    for (CodeColorizerFormat format : formats) {
+      byLang.put(format.getLanguageKey(), format);
+    }
+
+    LOG.debug("Code colorizer, supported languages: " + StringUtils.join(byLang.keySet(), ","));
+  }
+
+  /**
+   * Used when no plugin is defining some CodeColorizerFormat
+   */
+  public CodeColorizers() {
+    this(Lists.<CodeColorizerFormat>newArrayList());
+  }
+
+  public SyntaxHighlightingData toSyntaxHighlighting(File file, String encoding, String language) {
+    CodeColorizerFormat format = byLang.get(language);
+    List<Tokenizer> tokenizers;
+    if (format == null) {
+      tokenizers = null;
+    } else {
+      tokenizers = format.getTokenizers();
+    }
+    Reader reader = null;
+    try {
+      reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), encoding));
+      return new HighlightingRenderer().render(reader, tokenizers);
+    } catch (Exception e) {
+      throw new IllegalStateException("Unable to read source file for colorization", e);
+    } finally {
+      IOUtils.closeQuietly(reader);
+    }
+  }
+}
diff --git a/sonar-batch/src/main/java/org/sonar/batch/source/HighlightingCodeBuilder.java b/sonar-batch/src/main/java/org/sonar/batch/source/HighlightingCodeBuilder.java
new file mode 100644 (file)
index 0000000..1f52c21
--- /dev/null
@@ -0,0 +1,103 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.batch.source;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonar.api.batch.sensor.highlighting.TypeOfText;
+import org.sonar.batch.highlighting.SyntaxHighlightingData;
+import org.sonar.batch.highlighting.SyntaxHighlightingDataBuilder;
+import org.sonar.colorizer.HtmlCodeBuilder;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class HighlightingCodeBuilder extends HtmlCodeBuilder {
+
+  private static final Logger LOG = LoggerFactory.getLogger(HighlightingCodeBuilder.class);
+
+  private static final char BOM = '\uFEFF';
+
+  private SyntaxHighlightingDataBuilder highlightingBuilder = new SyntaxHighlightingDataBuilder();
+  private int currentOffset = 0;
+  private static final Pattern START_TAG_PATTERN = Pattern.compile("<span class=\"(.+)\">");
+  private static final Pattern END_TAG_PATTERN = Pattern.compile("</span>");
+  private int startOffset = -1;
+  private String cssClass;
+
+  @Override
+  public Appendable append(CharSequence csq) {
+    for (int i = 0; i < csq.length(); i++) {
+      append(csq.charAt(i));
+    }
+    return this;
+  }
+
+  @Override
+  public Appendable append(char c) {
+    if (c != BOM) {
+      currentOffset++;
+    }
+    return this;
+  }
+
+  @Override
+  public Appendable append(CharSequence csq, int start, int end) {
+    for (int i = start; i < end; i++) {
+      append(csq.charAt(i));
+    }
+    return this;
+  }
+
+  public void appendWithoutTransforming(String htmlTag) {
+    if (startOffset == -1) {
+      Matcher startMatcher = START_TAG_PATTERN.matcher(htmlTag);
+      if (startMatcher.matches()) {
+        startOffset = currentOffset;
+        cssClass = startMatcher.group(1);
+      } else {
+        LOG.warn("Expected to match highlighting start html tag but was: " + htmlTag);
+      }
+    } else {
+      Matcher endMatcher = END_TAG_PATTERN.matcher(htmlTag);
+      if (endMatcher.matches()) {
+        highlightingBuilder.registerHighlightingRule(startOffset, currentOffset, TypeOfText.forCssClass(cssClass));
+        startOffset = -1;
+      } else {
+        LOG.warn("Expected to match highlighting end html tag but was: " + htmlTag);
+      }
+    }
+  }
+
+  @Override
+  public String toString() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public StringBuilder getColorizedCode() {
+    throw new UnsupportedOperationException();
+  }
+
+  public SyntaxHighlightingData getHighlightingData() {
+    return highlightingBuilder.build();
+  }
+
+}
diff --git a/sonar-batch/src/main/java/org/sonar/batch/source/HighlightingRenderer.java b/sonar-batch/src/main/java/org/sonar/batch/source/HighlightingRenderer.java
new file mode 100644 (file)
index 0000000..e2a5faa
--- /dev/null
@@ -0,0 +1,44 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.batch.source;
+
+import org.sonar.batch.highlighting.SyntaxHighlightingData;
+import org.sonar.channel.Channel;
+import org.sonar.channel.CodeReader;
+import org.sonar.colorizer.HtmlCodeBuilder;
+import org.sonar.colorizer.TokenizerDispatcher;
+
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.List;
+
+public class HighlightingRenderer {
+
+  public SyntaxHighlightingData render(Reader code, List<? extends Channel<HtmlCodeBuilder>> tokenizers) {
+    List<Channel<HtmlCodeBuilder>> allTokenizers = new ArrayList<Channel<HtmlCodeBuilder>>();
+    HighlightingCodeBuilder codeBuilder = new HighlightingCodeBuilder();
+
+    allTokenizers.addAll(tokenizers);
+
+    new TokenizerDispatcher(allTokenizers).colorize(new CodeReader(code), codeBuilder);
+
+    return codeBuilder.getHighlightingData();
+  }
+}
index dcfdb8a14300562d1c3796416dad647637820081..7b7ff906d24c43f0196b210d2405eaa0c5f43d20 100644 (file)
@@ -45,6 +45,6 @@ public class DefaultHighlightingBuilderTest {
 
     ArgumentCaptor<SyntaxHighlightingData> argCaptor = ArgumentCaptor.forClass(SyntaxHighlightingData.class);
     verify(cache).setData(eq("myComponent"), eq(SnapshotDataTypes.SYNTAX_HIGHLIGHTING), argCaptor.capture());
-    assertThat(argCaptor.getValue().writeString()).isEqualTo("0,10,k;20,30,cppd;");
+    assertThat(argCaptor.getValue().writeString()).isEqualTo("0,10,k;20,30,cppd");
   }
 }
index c7ab820408c7f0640344a9a7ba38c32091670eab..6ab0249878e095ff1d82a98d70adb0ff6d6cff47 100644 (file)
@@ -44,6 +44,6 @@ public class SyntaxHighlightingDataTest {
       );
 
     String serializedRules = new SyntaxHighlightingData(orderedHighlightingRules).writeString();
-    assertThat(serializedRules).isEqualTo("0,10,cd;10,12,k;12,20,cd;24,38,k;24,65,cppd;42,50,k;");
+    assertThat(serializedRules).isEqualTo("0,10,cd;10,12,k;12,20,cd;24,38,k;24,65,cppd;42,50,k");
   }
 }
index 18bf6bed8898a293c32f60e5d47120fabfbc4838..453363df1ef3d2dd3a9824e0f5c8b79fc4bdbed3 100644 (file)
@@ -40,6 +40,7 @@ import org.sonar.batch.highlighting.SyntaxHighlightingData;
 import org.sonar.batch.highlighting.SyntaxHighlightingDataBuilder;
 import org.sonar.batch.scan.filesystem.InputPathCache;
 import org.sonar.batch.scan.measure.MeasureCache;
+import org.sonar.batch.source.CodeColorizers;
 import org.sonar.core.persistence.AbstractDaoTestCase;
 import org.sonar.core.source.SnapshotDataTypes;
 import org.sonar.core.source.db.SnapshotSourceDao;
@@ -86,7 +87,7 @@ public class SourcePersisterTest extends AbstractDaoTestCase {
     componentDataCache = mock(ComponentDataCache.class);
     sourcePersister = new SourcePersister(resourcePersister, new SnapshotSourceDao(getMyBatis()), inputPathCache,
       getMyBatis(), measureCache, componentDataCache, projectTree, system2,
-      resourceCache);
+      resourceCache, mock(CodeColorizers.class));
     Project project = new Project(PROJECT_KEY);
     project.setUuid("projectUuid");
     when(projectTree.getRootProject()).thenReturn(project);
diff --git a/sonar-batch/src/test/java/org/sonar/batch/source/CodeColorizersTest.java b/sonar-batch/src/test/java/org/sonar/batch/source/CodeColorizersTest.java
new file mode 100644 (file)
index 0000000..6056247
--- /dev/null
@@ -0,0 +1,129 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.batch.source;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.commons.io.FileUtils;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.sonar.api.web.CodeColorizerFormat;
+import org.sonar.batch.highlighting.SyntaxHighlightingData;
+import org.sonar.colorizer.CDocTokenizer;
+import org.sonar.colorizer.CppDocTokenizer;
+import org.sonar.colorizer.JavadocTokenizer;
+import org.sonar.colorizer.KeywordsTokenizer;
+import org.sonar.colorizer.StringTokenizer;
+import org.sonar.colorizer.Tokenizer;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.fest.assertions.Assertions.assertThat;
+
+public class CodeColorizersTest {
+
+  private static final String HIGHLIGHTING = "0,4,cppd;5,11,cppd;12,15,cppd;16,19,k;29,37,k;65,69,k;85,93,cd;98,102,k;112,114,s;120,124,k";
+  @Rule
+  public TemporaryFolder temp = new TemporaryFolder();
+
+  @Test
+  public void testConvertToHighlighting() throws Exception {
+    CodeColorizers codeColorizers = new CodeColorizers(Arrays.<CodeColorizerFormat>asList(new JavaScriptColorizerFormat()));
+
+    File jsFile = new File(this.getClass().getResource("CodeColorizersTest/Person.js").toURI());
+
+    SyntaxHighlightingData syntaxHighlighting = codeColorizers.toSyntaxHighlighting(jsFile, "UTF-8", "js");
+
+    assertThat(syntaxHighlighting.writeString()).isEqualTo(HIGHLIGHTING);
+
+  }
+
+  @Test
+  public void testConvertToHighlightingIgnoreBOM() throws Exception {
+    CodeColorizers codeColorizers = new CodeColorizers(Arrays.<CodeColorizerFormat>asList(new JavaScriptColorizerFormat()));
+
+    File fileWithBom = temp.newFile();
+    FileUtils.write(fileWithBom, "\uFEFF");
+    File jsFile = new File(this.getClass().getResource("CodeColorizersTest/Person.js").toURI());
+    FileUtils.write(fileWithBom, FileUtils.readFileToString(jsFile), true);
+
+    SyntaxHighlightingData syntaxHighlighting = codeColorizers.toSyntaxHighlighting(fileWithBom, "UTF-8", "js");
+
+    assertThat(syntaxHighlighting.writeString()).isEqualTo(HIGHLIGHTING);
+
+  }
+
+  public static class JavaScriptColorizerFormat extends CodeColorizerFormat {
+
+    public JavaScriptColorizerFormat() {
+      super("js");
+    }
+
+    @Override
+    public List<Tokenizer> getTokenizers() {
+      return ImmutableList.<Tokenizer>of(
+        new StringTokenizer("<span class=\"s\">", "</span>"),
+        new CDocTokenizer("<span class=\"cd\">", "</span>"),
+        new JavadocTokenizer("<span class=\"cppd\">", "</span>"),
+        new CppDocTokenizer("<span class=\"cppd\">", "</span>"),
+        new KeywordsTokenizer("<span class=\"k\">", "</span>", "null",
+          "true",
+          "false",
+          "break",
+          "case",
+          "catch",
+          "class",
+          "continue",
+          "debugger",
+          "default",
+          "delete",
+          "do",
+          "extends",
+          "else",
+          "finally",
+          "for",
+          "function",
+          "if",
+          "import",
+          "in",
+          "instanceof",
+          "new",
+          "return",
+          "super",
+          "switch",
+          "this",
+          "throw",
+          "try",
+          "typeof",
+          "var",
+          "void",
+          "while",
+          "with",
+          "yield",
+          "const",
+          "enum",
+          "export"));
+    }
+
+  }
+
+}
index efa8024c6a9d08cf5c7d198746ab5b64b43a8dd1..da0966aee6e823b9a029763a05164e5fe4e4d67b 100644 (file)
@@ -62,6 +62,6 @@ public class DefaultHighlightableTest {
 
     ArgumentCaptor<SyntaxHighlightingData> argCaptor = ArgumentCaptor.forClass(SyntaxHighlightingData.class);
     verify(cache).setData(eq("myComponent"), eq(SnapshotDataTypes.SYNTAX_HIGHLIGHTING), argCaptor.capture());
-    assertThat(argCaptor.getValue().writeString()).isEqualTo("0,10,k;20,30,cppd;");
+    assertThat(argCaptor.getValue().writeString()).isEqualTo("0,10,k;20,30,cppd");
   }
 }
diff --git a/sonar-batch/src/test/resources/org/sonar/batch/source/CodeColorizersTest/Person.js b/sonar-batch/src/test/resources/org/sonar/batch/source/CodeColorizersTest/Person.js
new file mode 100644 (file)
index 0000000..fc36e5a
--- /dev/null
@@ -0,0 +1,8 @@
+/** 
+ * Doc
+ */
+var Person = function(first, last, middle) {
+    this.first = first; // First
+    this.middle = '';
+    this.last = 1;
+};
index 26544ad5c50cd33060e6fdb321e138ffa57532e9..bd17c39315500bb17c5b18093cbbe9adf485626b 100644 (file)
  */
 package org.sonar.api.web;
 
-import java.util.List;
-
 import com.google.common.base.Objects;
 import org.sonar.api.ServerExtension;
+import org.sonar.api.source.Highlightable;
+import org.sonar.api.task.TaskExtension;
 import org.sonar.colorizer.Tokenizer;
 
+import java.util.List;
+
 /**
- * Extend the library sonar-colorizer to support new languages. By default only Java sources are colorized in Sonar.
+ * Extend the library sonar-colorizer to support new languages.
  * 
  * @since 1.12
+ * @deprecated since 5.1 use {@link Highlightable} API
  */
-public abstract class CodeColorizerFormat implements ServerExtension {
+public abstract class CodeColorizerFormat implements ServerExtension, TaskExtension {
 
   private String languageKey;
 
@@ -57,7 +60,7 @@ public abstract class CodeColorizerFormat implements ServerExtension {
     if (this == o) {
       return true;
     }
-    if ( !(o instanceof CodeColorizerFormat)) {
+    if (!(o instanceof CodeColorizerFormat)) {
       return false;
     }