Rewrite markdown rendering to blackfriday v2 and rewrite orgmode rendering to go-org (#8560)

* Rewrite markdown rendering to blackfriday v2.0

* Fix style

* Fix go mod with golang 1.13

* Fix blackfriday v2 import

* Inital orgmode renderer migration to go-org

* Vendor go-org dependency

* Ignore errors :/

* Update go-org to latest version

* Update test

* Fix go-org test

* Remove unneeded code

* Fix comments

* Fix markdown test

* Fix blackfriday regression rendering HTML block

2 files changed, 111 insertions, 106 deletions

diff --git a/modules/markup/markdown/markdown.go b/modules/markup/markdown/markdown.go
index d9fc768891..ff78d7ea3a 100644
--- a/modules/markup/markdown/markdown.go
+++ b/modules/markup/markdown/markdown.go
@@ -7,13 +7,14 @@ package markdown
 
 import (
 	"bytes"
+	"io"
 	"strings"
 
 	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
 
-	"github.com/russross/blackfriday"
+	"github.com/russross/blackfriday/v2"
 )
 
 // Renderer is a extended version of underlying render object.
@@ -25,134 +26,138 @@ type Renderer struct {
 
 var byteMailto = []byte("mailto:")
 
-// Link defines how formal links should be processed to produce corresponding HTML elements.
-func (r *Renderer) Link(out *bytes.Buffer, link []byte, title []byte, content []byte) {
-	// special case: this is not a link, a hash link or a mailto:, so it's a
-	// relative URL
-	if len(link) > 0 && !markup.IsLink(link) &&
-		link[0] != '#' && !bytes.HasPrefix(link, byteMailto) {
-		lnk := string(link)
-		if r.IsWiki {
-			lnk = util.URLJoin("wiki", lnk)
-		}
-		mLink := util.URLJoin(r.URLPrefix, lnk)
-		link = []byte(mLink)
-	}
-
-	if len(content) > 10 && string(content[0:9]) == "<a href=\"" && bytes.Contains(content[9:], []byte("<img")) {
-		// Image with link case: markdown `[![]()]()`
-		// If the content is an image, then we change the original href around it
-		// which points to itself to a new address "link"
-		rightQuote := bytes.Index(content[9:], []byte("\""))
-		content = bytes.Replace(content, content[9:9+rightQuote], link, 1)
-		out.Write(content)
-	} else {
-		r.Renderer.Link(out, link, title, content)
-	}
+var htmlEscaper = [256][]byte{
+	'&': []byte("&amp;"),
+	'<': []byte("&lt;"),
+	'>': []byte("&gt;"),
+	'"': []byte("&quot;"),
 }
 
-// List renders markdown bullet or digit lists to HTML
-func (r *Renderer) List(out *bytes.Buffer, text func() bool, flags int) {
-	marker := out.Len()
-	if out.Len() > 0 {
-		out.WriteByte('\n')
-	}
-
-	if flags&blackfriday.LIST_TYPE_DEFINITION != 0 {
-		out.WriteString("<dl>")
-	} else if flags&blackfriday.LIST_TYPE_ORDERED != 0 {
-		out.WriteString("<ol class='ui list'>")
-	} else {
-		out.WriteString("<ul class='ui list'>")
-	}
-	if !text() {
-		out.Truncate(marker)
-		return
+func escapeHTML(w io.Writer, s []byte) {
+	var start, end int
+	for end < len(s) {
+		escSeq := htmlEscaper[s[end]]
+		if escSeq != nil {
+			_, _ = w.Write(s[start:end])
+			_, _ = w.Write(escSeq)
+			start = end + 1
+		}
+		end++
 	}
-	if flags&blackfriday.LIST_TYPE_DEFINITION != 0 {
-		out.WriteString("</dl>\n")
-	} else if flags&blackfriday.LIST_TYPE_ORDERED != 0 {
-		out.WriteString("</ol>\n")
-	} else {
-		out.WriteString("</ul>\n")
+	if start < len(s) && end <= len(s) {
+		_, _ = w.Write(s[start:end])
 	}
 }
 
-// ListItem defines how list items should be processed to produce corresponding HTML elements.
-func (r *Renderer) ListItem(out *bytes.Buffer, text []byte, flags int) {
-	// Detect procedures to draw checkboxes.
-	prefix := ""
-	if bytes.HasPrefix(text, []byte("<p>")) {
-		prefix = "<p>"
-	}
-	switch {
-	case bytes.HasPrefix(text, []byte(prefix+"[ ] ")):
-		text = append([]byte(`<span class="ui fitted disabled checkbox"><input type="checkbox" disabled="disabled" /><label /></span>`), text[3+len(prefix):]...)
-		if prefix != "" {
-			text = bytes.Replace(text, []byte(prefix), []byte{}, 1)
+// RenderNode is a default renderer of a single node of a syntax tree. For
+// block nodes it will be called twice: first time with entering=true, second
+// time with entering=false, so that it could know when it's working on an open
+// tag and when on close. It writes the result to w.
+//
+// The return value is a way to tell the calling walker to adjust its walk
+// pattern: e.g. it can terminate the traversal by returning Terminate. Or it
+// can ask the walker to skip a subtree of this node by returning SkipChildren.
+// The typical behavior is to return GoToNext, which asks for the usual
+// traversal to the next node.
+func (r *Renderer) RenderNode(w io.Writer, node *blackfriday.Node, entering bool) blackfriday.WalkStatus {
+	switch node.Type {
+	case blackfriday.Image:
+		prefix := r.URLPrefix
+		if r.IsWiki {
+			prefix = util.URLJoin(prefix, "wiki", "raw")
 		}
-	case bytes.HasPrefix(text, []byte(prefix+"[x] ")):
-		text = append([]byte(`<span class="ui checked fitted disabled checkbox"><input type="checkbox" checked="" disabled="disabled" /><label /></span>`), text[3+len(prefix):]...)
-		if prefix != "" {
-			text = bytes.Replace(text, []byte(prefix), []byte{}, 1)
+		prefix = strings.Replace(prefix, "/src/", "/media/", 1)
+		link := node.LinkData.Destination
+		if len(link) > 0 && !markup.IsLink(link) {
+			lnk := string(link)
+			lnk = util.URLJoin(prefix, lnk)
+			lnk = strings.Replace(lnk, " ", "+", -1)
+			link = []byte(lnk)
+		}
+		node.LinkData.Destination = link
+		// Render link around image only if parent is not link already
+		if node.Parent != nil && node.Parent.Type != blackfriday.Link {
+			if entering {
+				_, _ = w.Write([]byte(`<a href="`))
+				escapeHTML(w, link)
+				_, _ = w.Write([]byte(`">`))
+				return r.Renderer.RenderNode(w, node, entering)
+			}
+			s := r.Renderer.RenderNode(w, node, entering)
+			_, _ = w.Write([]byte(`</a>`))
+			return s
+		}
+		return r.Renderer.RenderNode(w, node, entering)
+	case blackfriday.Link:
+		// special case: this is not a link, a hash link or a mailto:, so it's a
+		// relative URL
+		link := node.LinkData.Destination
+		if len(link) > 0 && !markup.IsLink(link) &&
+			link[0] != '#' && !bytes.HasPrefix(link, byteMailto) &&
+			node.LinkData.Footnote == nil {
+			lnk := string(link)
+			if r.IsWiki {
+				lnk = util.URLJoin("wiki", lnk)
+			}
+			link = []byte(util.URLJoin(r.URLPrefix, lnk))
+		}
+		node.LinkData.Destination = link
+		return r.Renderer.RenderNode(w, node, entering)
+	case blackfriday.Text:
+		isListItem := false
+		for n := node.Parent; n != nil; n = n.Parent {
+			if n.Type == blackfriday.Item {
+				isListItem = true
+				break
+			}
+		}
+		if isListItem {
+			text := node.Literal
+			switch {
+			case bytes.HasPrefix(text, []byte("[ ] ")):
+				_, _ = w.Write([]byte(`<span class="ui fitted disabled checkbox"><input type="checkbox" disabled="disabled" /><label /></span>`))
+				text = text[3:]
+			case bytes.HasPrefix(text, []byte("[x] ")):
+				_, _ = w.Write([]byte(`<span class="ui checked fitted disabled checkbox"><input type="checkbox" checked="" disabled="disabled" /><label /></span>`))
+				text = text[3:]
+			}
+			node.Literal = text
 		}
 	}
-	r.Renderer.ListItem(out, text, flags)
-}
-
-// Image defines how images should be processed to produce corresponding HTML elements.
-func (r *Renderer) Image(out *bytes.Buffer, link []byte, title []byte, alt []byte) {
-	prefix := r.URLPrefix
-	if r.IsWiki {
-		prefix = util.URLJoin(prefix, "wiki", "raw")
-	}
-	prefix = strings.Replace(prefix, "/src/", "/media/", 1)
-	if len(link) > 0 && !markup.IsLink(link) {
-		lnk := string(link)
-		lnk = util.URLJoin(prefix, lnk)
-		lnk = strings.Replace(lnk, " ", "+", -1)
-		link = []byte(lnk)
-	}
-
-	// Put a link around it pointing to itself by default
-	out.WriteString(`<a href="`)
-	out.Write(link)
-	out.WriteString(`">`)
-	r.Renderer.Image(out, link, title, alt)
-	out.WriteString("</a>")
+	return r.Renderer.RenderNode(w, node, entering)
 }
 
 const (
 	blackfridayExtensions = 0 |
-		blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
-		blackfriday.EXTENSION_TABLES |
-		blackfriday.EXTENSION_FENCED_CODE |
-		blackfriday.EXTENSION_STRIKETHROUGH |
-		blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK |
-		blackfriday.EXTENSION_DEFINITION_LISTS |
-		blackfriday.EXTENSION_FOOTNOTES |
-		blackfriday.EXTENSION_HEADER_IDS |
-		blackfriday.EXTENSION_AUTO_HEADER_IDS
+		blackfriday.NoIntraEmphasis |
+		blackfriday.Tables |
+		blackfriday.FencedCode |
+		blackfriday.Strikethrough |
+		blackfriday.NoEmptyLineBeforeBlock |
+		blackfriday.DefinitionLists |
+		blackfriday.Footnotes |
+		blackfriday.HeadingIDs |
+		blackfriday.AutoHeadingIDs
 	blackfridayHTMLFlags = 0 |
-		blackfriday.HTML_SKIP_STYLE |
-		blackfriday.HTML_OMIT_CONTENTS |
-		blackfriday.HTML_USE_SMARTYPANTS
+		blackfriday.Smartypants
 )
 
 // RenderRaw renders Markdown to HTML without handling special links.
 func RenderRaw(body []byte, urlPrefix string, wikiMarkdown bool) []byte {
 	renderer := &Renderer{
-		Renderer:  blackfriday.HtmlRenderer(blackfridayHTMLFlags, "", ""),
+		Renderer: blackfriday.NewHTMLRenderer(blackfriday.HTMLRendererParameters{
+			Flags: blackfridayHTMLFlags,
+		}),
 		URLPrefix: urlPrefix,
 		IsWiki:    wikiMarkdown,
 	}
 
 	exts := blackfridayExtensions
 	if setting.Markdown.EnableHardLineBreak {
-		exts |= blackfriday.EXTENSION_HARD_LINE_BREAK
+		exts |= blackfriday.HardLineBreak
 	}
 
-	body = blackfriday.Markdown(body, renderer, exts)
+	body = blackfriday.Run(body, blackfriday.WithRenderer(renderer), blackfriday.WithExtensions(exts))
 	return markup.SanitizeBytes(body)
 }
 
diff --git a/modules/markup/markdown/markdown_test.go b/modules/markup/markdown/markdown_test.go
index 669b49367e..b29f870ce5 100644
--- a/modules/markup/markdown/markdown_test.go
+++ b/modules/markup/markdown/markdown_test.go
@@ -166,13 +166,13 @@ func testAnswers(baseURLContent, baseURLImages string) []string {
 <h3 id="footnotes">Footnotes</h3>
 
 <p>Here is a simple footnote,<sup id="fnref:1"><a href="#fn:1" rel="nofollow">1</a></sup> and here is a longer one.<sup id="fnref:bignote"><a href="#fn:bignote" rel="nofollow">2</a></sup></p>
+
 <div>
 
 <hr/>
 
 <ol>
-<li id="fn:1">This is the first footnote.
-</li>
+<li id="fn:1">This is the first footnote.</li>
 
 <li id="fn:bignote"><p>Here is one with multiple paragraphs and code.</p>
 
@@ -180,9 +180,9 @@ func testAnswers(baseURLContent, baseURLImages string) []string {
 
 <p><code>{ my code }</code></p>
 
-<p>Add as many paragraphs as you like.</p>
-</li>
+<p>Add as many paragraphs as you like.</p></li>
 </ol>
+
 </div>
 `,
 	}