Adds CommonMark Markdown (GitHub Flavored) as third text formatting option (#32424).

Patch by Jens Krämer.

git-svn-id: http://svn.redmine.org/redmine/trunk@21156 e93f8b46-1217-0410-a6f0-8f06a7374b81

9 files changed, 449 insertions, 0 deletions

diff --git a/lib/redmine.rb b/lib/redmine.rb
index 1328935da..937504892 100644
--- a/lib/redmine.rb
+++ b/lib/redmine.rb
@@ -29,6 +29,11 @@ begin
 rescue LoadError
   # Redcarpet is not available
 end
+begin
+  require 'commonmarker' unless Object.const_defined?(:CommonMarker)
+rescue LoadError
+  # CommonMarker is not available
+end
 
 require 'redmine/acts/positioned'
 
@@ -442,6 +447,9 @@ end
 Redmine::WikiFormatting.map do |format|
   format.register :textile
   format.register :markdown if Object.const_defined?(:Redcarpet)
+  if Object.const_defined?(:CommonMarker)
+    format.register :common_mark, label: 'CommonMark Markdown (GitHub Flavored)'
+  end
 end
 
 ActionView::Template.register_template_handler :rsb, Redmine::Views::ApiTemplateHandler
diff --git a/lib/redmine/wiki_formatting/common_mark/external_links_filter.rb b/lib/redmine/wiki_formatting/common_mark/external_links_filter.rb
new file mode 100644
index 000000000..023593b5a
--- /dev/null
+++ b/lib/redmine/wiki_formatting/common_mark/external_links_filter.rb
@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+# Redmine - project management software
+# Copyright (C) 2006-2021  Jean-Philippe Lang
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+require 'uri'
+
+module Redmine
+  module WikiFormatting
+    module CommonMark
+      # adds class="external" to external links, and class="email" to mailto
+      # links
+      class ExternalLinksFilter < HTML::Pipeline::Filter
+        def call
+          doc.search("a").each do |node|
+            url = node["href"]
+            next unless url
+            next if url.starts_with?("/") || url.starts_with?("#") || !url.include?(':')
+
+            scheme = URI.parse(url).scheme
+            next if scheme.blank?
+
+            klass = node["class"].presence
+            node["class"] = [
+              klass,
+              (scheme == "mailto" ? "email" : "external")
+            ].compact.join " "
+          end
+          doc
+        end
+      end
+    end
+  end
+end
diff --git a/lib/redmine/wiki_formatting/common_mark/fixup_auto_links_filter.rb b/lib/redmine/wiki_formatting/common_mark/fixup_auto_links_filter.rb
new file mode 100644
index 000000000..94120bccf
--- /dev/null
+++ b/lib/redmine/wiki_formatting/common_mark/fixup_auto_links_filter.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# Redmine - project management software
+# Copyright (C) 2006-2021  Jean-Philippe Lang
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+module Redmine
+  module WikiFormatting
+    module CommonMark
+      # fixes:
+      # - autolinked email addresses that are actually references to users:
+      #   user:<a href="mailto:user@example.org">user@example.org</a>
+      #   @<a href="mailto:user@example.org">user@example.org</a>
+      # - autolinked hi res image names that look like email addresses:
+      #   <a href="mailto:printscreen@2x.png">printscreen@2x.png</a>
+      class FixupAutoLinksFilter < HTML::Pipeline::Filter
+        USER_LINK_PREFIX = /(@|user:)\z/.freeze
+        HIRES_IMAGE = /.+@\dx\.(bmp|gif|jpg|jpe|jpeg|png)\z/.freeze
+
+        def call
+          doc.search("a").each do |node|
+            unless (url = node['href']) && url.starts_with?('mailto:')
+              next
+            end
+
+            if ((p = node.previous) && p.text? &&
+                p.text =~(USER_LINK_PREFIX)) ||
+               (node.text =~ HIRES_IMAGE)
+
+              node.replace node.text
+            end
+          end
+          doc
+        end
+      end
+    end
+  end
+end
diff --git a/lib/redmine/wiki_formatting/common_mark/formatter.rb b/lib/redmine/wiki_formatting/common_mark/formatter.rb
new file mode 100644
index 000000000..6a9c95c8e
--- /dev/null
+++ b/lib/redmine/wiki_formatting/common_mark/formatter.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+# Redmine - project management software
+# Copyright (C) 2006-2021  Jean-Philippe Lang
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+require 'html/pipeline'
+
+module Redmine
+  module WikiFormatting
+    module CommonMark
+      # configuration of the rendering pipeline
+      PIPELINE_CONFIG = {
+        # https://github.com/gjtorikian/commonmarker#extensions
+        commonmarker_extensions: [
+          :table,
+          :strikethrough,
+          :tagfilter,
+          :autolink
+        ].freeze,
+
+        # https://github.com/gjtorikian/commonmarker#parse-options
+        commonmarker_parse_options: [
+          :FOOTNOTES,
+          :STRIKETHROUGH_DOUBLE_TILDE,
+          :UNSAFE,
+          :VALIDATE_UTF8
+        ].freeze,
+
+        # https://github.com/gjtorikian/commonmarker#render-options
+        commonmarker_render_options: [
+          :HARDBREAKS,
+          :UNSAFE
+        ].freeze,
+      }.freeze
+
+      MarkdownPipeline = HTML::Pipeline.new [
+        MarkdownFilter,
+        SanitizationFilter,
+        SyntaxHighlightFilter,
+        FixupAutoLinksFilter,
+        ExternalLinksFilter,
+      ], PIPELINE_CONFIG
+
+      class Formatter < Redmine::WikiFormatting::Markdown::Formatter
+        def to_html(*args)
+          result = MarkdownPipeline.call @text
+          result[:output].to_s
+        end
+      end
+    end
+  end
+end
diff --git a/lib/redmine/wiki_formatting/common_mark/helper.rb b/lib/redmine/wiki_formatting/common_mark/helper.rb
new file mode 100644
index 000000000..471835708
--- /dev/null
+++ b/lib/redmine/wiki_formatting/common_mark/helper.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+# Redmine - project management software
+# Copyright (C) 2006-2021  Jean-Philippe Lang
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+module Redmine
+  module WikiFormatting
+    module CommonMark
+      module Helper
+        include Redmine::WikiFormatting::Markdown::Helper
+
+        def wikitoolbar_for(field_id, preview_url = preview_text_path)
+          heads_for_wiki_formatter
+          help_file = "/help/#{current_language.to_s.downcase}/wiki_syntax_common_mark.html"
+          # fall back to the english help page if there is none for the current
+          # language
+          unless File.readable? Rails.root.join("public", help_file)
+            help_file = "/help/en/wiki_syntax_common_mark.html"
+          end
+          url = "#{Redmine::Utils.relative_url_root}#{help_file}"
+          javascript_tag(
+            "var wikiToolbar = new jsToolBar(document.getElementById('#{field_id}')); " \
+            "wikiToolbar.setHelpLink('#{escape_javascript url}'); " \
+            "wikiToolbar.setPreviewUrl('#{escape_javascript preview_url}'); " \
+            "wikiToolbar.draw();"
+          )
+        end
+
+        # removes the 'underline' icon from the markdown toolbar since there
+        # is no such thing in CommonMark
+        def heads_for_wiki_formatter
+          unless @common_mark_heads_for_wiki_formatter_included
+            super
+            content_for :header_tags do
+              javascript_tag(%[delete jsToolBar.prototype.elements.ins;])
+            end
+            @common_mark_heads_for_wiki_formatter_included = true
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/redmine/wiki_formatting/common_mark/html_parser.rb b/lib/redmine/wiki_formatting/common_mark/html_parser.rb
new file mode 100644
index 000000000..707829fb6
--- /dev/null
+++ b/lib/redmine/wiki_formatting/common_mark/html_parser.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# Redmine - project management software
+# Copyright (C) 2006-2021  Jean-Philippe Lang
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+module Redmine
+  module WikiFormatting
+    module CommonMark
+      HtmlParser = Redmine::WikiFormatting::Markdown::HtmlParser
+    end
+  end
+end
diff --git a/lib/redmine/wiki_formatting/common_mark/markdown_filter.rb b/lib/redmine/wiki_formatting/common_mark/markdown_filter.rb
new file mode 100644
index 000000000..b93b35b68
--- /dev/null
+++ b/lib/redmine/wiki_formatting/common_mark/markdown_filter.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+# Redmine - project management software
+# Copyright (C) 2006-2021  Jean-Philippe Lang
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+module Redmine
+  module WikiFormatting
+    module CommonMark
+      # Converts Markdown to HTML using CommonMarker
+      #
+      # We do not use the stock HTML::Pipeline::MarkdownFilter because this
+      # does not allow for straightforward configuration of render and parsing
+      # options
+      class MarkdownFilter < HTML::Pipeline::TextFilter
+        def initialize(text, context = nil, result = nil)
+          super text, context, result
+          @text = @text.delete "\r"
+        end
+
+        def call
+          doc = CommonMarker.render_doc(@text, parse_options, extensions)
+          html = doc.to_html render_options, extensions
+          html.rstrip!
+          html
+        end
+
+        private
+
+        def extensions
+          context.fetch :commonmarker_extensions, []
+        end
+
+        def parse_options
+          context.fetch :commonmarker_parse_options, :DEFAULT
+        end
+
+        def render_options
+          context.fetch :commonmarker_render_options, :DEFAULT
+        end
+      end
+    end
+  end
+end
diff --git a/lib/redmine/wiki_formatting/common_mark/sanitization_filter.rb b/lib/redmine/wiki_formatting/common_mark/sanitization_filter.rb
new file mode 100644
index 000000000..a76201dfd
--- /dev/null
+++ b/lib/redmine/wiki_formatting/common_mark/sanitization_filter.rb
@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+# Redmine - project management software
+# Copyright (C) 2006-2021  Jean-Philippe Lang
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+module Redmine
+  module WikiFormatting
+    module CommonMark
+      # sanitizes rendered HTML using the Sanitize gem
+      class SanitizationFilter < HTML::Pipeline::SanitizationFilter
+        def whitelist
+          @@whitelist ||= customize_whitelist(super.deep_dup)
+        end
+
+        private
+
+        # customizes the whitelist defined in
+        # https://github.com/jch/html-pipeline/blob/master/lib/html/pipeline/sanitization_filter.rb
+        def customize_whitelist(whitelist)
+          # Disallow `name` attribute globally, allow on `a`
+          whitelist[:attributes][:all].delete("name")
+          whitelist[:attributes]["a"].push("name")
+
+          # allow class on code tags (this holds the language info from fenced
+          # code bocks and has the format language-foo)
+          whitelist[:attributes]["code"] = %w(class)
+          whitelist[:transformers].push lambda{|env|
+            node = env[:node]
+            return unless node.name == "code"
+            return unless node.has_attribute?("class")
+
+            unless /\Alanguage-(\w+)\z/.match?(node["class"])
+              node.remove_attribute("class")
+            end
+          }
+
+          # Allow table cell alignment by style attribute
+          #
+          # Only necessary if we used the TABLE_PREFER_STYLE_ATTRIBUTES
+          # commonmarker option (which we do not, currently).
+          # By default, the align attribute is used (which is allowed on all
+          # elements).
+          # whitelist[:attributes]["th"] = %w(style)
+          # whitelist[:attributes]["td"] = %w(style)
+          # whitelist[:css] = { properties: ["text-align"] }
+
+          # Allow `id` in a and li elements for footnotes
+          # and remove any `id` properties not matching for footnotes
+          whitelist[:attributes]["a"].push "id"
+          whitelist[:attributes]["li"] = %w(id)
+          whitelist[:transformers].push lambda{|env|
+            node = env[:node]
+            return unless node.name == "a" || node.name == "li"
+            return unless node.has_attribute?("id")
+            return if node.name == "a" && node["id"] =~ /\Afnref\d+\z/
+            return if node.name == "li" && node["id"] =~ /\Afn\d+\z/
+
+            node.remove_attribute("id")
+          }
+
+          # allow the same set of URL schemes for links as is the default in
+          # Redmine::Helpers::URL#uri_with_safe_scheme?
+          whitelist[:protocols]["a"]["href"] = [
+            'http', 'https', 'ftp', 'mailto', :relative
+          ]
+
+          whitelist
+        end
+      end
+    end
+  end
+end
diff --git a/lib/redmine/wiki_formatting/common_mark/syntax_highlight_filter.rb b/lib/redmine/wiki_formatting/common_mark/syntax_highlight_filter.rb
new file mode 100644
index 000000000..a027e6a17
--- /dev/null
+++ b/lib/redmine/wiki_formatting/common_mark/syntax_highlight_filter.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# Redmine - project management software
+# Copyright (C) 2006-2021  Jean-Philippe Lang
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+module Redmine
+  module WikiFormatting
+    module CommonMark
+      # Redmine Syntax highlighting for <pre><code class="language-foo">
+      # blocks as generated by commonmarker
+      class SyntaxHighlightFilter < HTML::Pipeline::Filter
+        def call
+          doc.search("pre > code").each do |node|
+            next unless lang = node["class"].presence
+            next unless lang =~ /\Alanguage-(\w+)\z/
+
+            lang = $1
+            text = node.inner_text
+
+            if Redmine::SyntaxHighlighting.language_supported?(lang)
+              html = Redmine::SyntaxHighlighting.highlight_by_language(text, lang)
+              next if html.nil?
+
+              node.inner_html = html
+              node["class"] = "#{lang} syntaxhl"
+            else
+              # unsupported language, strip out the code tag
+              node.parent.inner_html = text
+            end
+          end
+          doc
+        end
+      end
+    end
+  end
+end