Make markdown as an independent module

6 files changed, 127 insertions, 119 deletions

diff --git a/modules/avatar/avatar.go b/modules/avatar/avatar.go
index 3d885eb3d4..e9d0c1282e 100644
--- a/modules/avatar/avatar.go
+++ b/modules/avatar/avatar.go
@@ -16,6 +16,8 @@ import (
 
 const _RANDOM_AVATAR_SIZE = 200
 
+// RandomImage generates and returns a random avatar image unique to input data
+// in custom size (height and width).
 func RandomImageSize(size int, data []byte) (image.Image, error) {
 	randExtent := len(palette.WebSafe) - 32
 	rand.Seed(time.Now().UnixNano())
@@ -34,8 +36,8 @@ func RandomImageSize(size int, data []byte) (image.Image, error) {
 	return imgMaker.Make(data), nil
 }
 
-// RandomImage generates and returns a random avatar image.
-// The data should normally be the []byte type of email address.
+// RandomImage generates and returns a random avatar image unique to input data
+// in default size (height and width).
 func RandomImage(data []byte) (image.Image, error) {
 	return RandomImageSize(_RANDOM_AVATAR_SIZE, data)
 }
diff --git a/modules/base/base.go b/modules/base/base.go
index 45e2151e38..8ba211aa7a 100644
--- a/modules/base/base.go
+++ b/modules/base/base.go
@@ -4,27 +4,8 @@
 
 package base
 
-import (
-	"os"
-	"os/exec"
-	"path/filepath"
-)
-
 const DOC_URL = "https://github.com/gogits/go-gogs-client/wiki"
 
 type (
 	TplName string
 )
-
-// ExecPath returns the executable path.
-func ExecPath() (string, error) {
-	file, err := exec.LookPath(os.Args[0])
-	if err != nil {
-		return "", err
-	}
-	p, err := filepath.Abs(file)
-	if err != nil {
-		return "", err
-	}
-	return p, nil
-}
diff --git a/modules/base/tool.go b/modules/base/tool.go
index 811a76960c..bc6ff81a14 100644
--- a/modules/base/tool.go
+++ b/modules/base/tool.go
@@ -15,14 +15,14 @@ import (
 	"hash"
 	"html/template"
 	"math"
-	"regexp"
+	"net/http"
 	"strings"
 	"time"
+	"unicode"
 	"unicode/utf8"
 
 	"github.com/Unknwon/com"
 	"github.com/Unknwon/i18n"
-	"github.com/microcosm-cc/bluemonday"
 
 	"github.com/gogits/chardet"
 
@@ -30,20 +30,6 @@ import (
 	"github.com/gogits/gogs/modules/setting"
 )
 
-var Sanitizer = bluemonday.UGCPolicy()
-
-func BuildSanitizer() {
-	// Normal markdown-stuff
-	Sanitizer.AllowAttrs("class").Matching(regexp.MustCompile(`[\p{L}\p{N}\s\-_',:\[\]!\./\\\(\)&]*`)).OnElements("code")
-
-	// Checkboxes
-	Sanitizer.AllowAttrs("type").Matching(regexp.MustCompile(`^checkbox$`)).OnElements("input")
-	Sanitizer.AllowAttrs("checked", "disabled").OnElements("input")
-
-	// Custom URL-Schemes
-	Sanitizer.AllowURLSchemes(setting.Markdown.CustomURLSchemes...)
-}
-
 // EncodeMD5 encodes string to md5 hex value.
 func EncodeMD5(str string) string {
 	m := md5.New()
@@ -504,3 +490,25 @@ func Int64sToMap(ints []int64) map[int64]bool {
 	}
 	return m
 }
+
+// IsLetter reports whether the rune is a letter (category L).
+// https://github.com/golang/go/blob/master/src/go/scanner/scanner.go#L257
+func IsLetter(ch rune) bool {
+	return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || ch >= 0x80 && unicode.IsLetter(ch)
+}
+
+func IsTextFile(data []byte) (string, bool) {
+	contentType := http.DetectContentType(data)
+	if strings.Index(contentType, "text/") != -1 {
+		return contentType, true
+	}
+	return contentType, false
+}
+
+func IsImageFile(data []byte) (string, bool) {
+	contentType := http.DetectContentType(data)
+	if strings.Index(contentType, "image/") != -1 {
+		return contentType, true
+	}
+	return contentType, false
+}
diff --git a/modules/mailer/mail.go b/modules/mailer/mail.go
index ca5618f1bc..53455fc012 100644
--- a/modules/mailer/mail.go
+++ b/modules/mailer/mail.go
@@ -14,6 +14,7 @@ import (
 	"github.com/gogits/gogs/models"
 	"github.com/gogits/gogs/modules/base"
 	"github.com/gogits/gogs/modules/log"
+	"github.com/gogits/gogs/modules/markdown"
 	"github.com/gogits/gogs/modules/setting"
 )
 
@@ -126,7 +127,7 @@ func SendIssueNotifyMail(u, owner *models.User, repo *models.Repository, issue *
 
 	subject := fmt.Sprintf("[%s] %s (#%d)", repo.Name, issue.Name, issue.Index)
 	content := fmt.Sprintf("%s<br>-<br> <a href=\"%s%s/%s/issues/%d\">View it on Gogs</a>.",
-		base.RenderSpecialLink([]byte(strings.Replace(issue.Content, "\n", "<br>", -1)), owner.Name+"/"+repo.Name, repo.ComposeMetas()),
+		markdown.RenderSpecialLink([]byte(strings.Replace(issue.Content, "\n", "<br>", -1)), owner.Name+"/"+repo.Name, repo.ComposeMetas()),
 		setting.AppUrl, owner.Name, repo.Name, issue.Index)
 	msg := NewMessage(tos, subject, content)
 	msg.Info = fmt.Sprintf("Subject: %s, issue notify", subject)
@@ -149,7 +150,7 @@ func SendIssueMentionMail(r macaron.Render, u, owner *models.User,
 	data["IssueLink"] = fmt.Sprintf("%s/%s/issues/%d", owner.Name, repo.Name, issue.Index)
 	data["Subject"] = subject
 	data["ActUserName"] = u.DisplayName()
-	data["Content"] = string(base.RenderSpecialLink([]byte(issue.Content), owner.Name+"/"+repo.Name, repo.ComposeMetas()))
+	data["Content"] = string(markdown.RenderSpecialLink([]byte(issue.Content), owner.Name+"/"+repo.Name, repo.ComposeMetas()))
 
 	body, err := r.HTMLString(string(NOTIFY_MENTION), data)
 	if err != nil {
diff --git a/modules/base/markdown.go b/modules/markdown/markdown.go
index 10158edd32..aad0553afe 100644
--- a/modules/base/markdown.go
+++ b/modules/markdown/markdown.go
@@ -2,41 +2,51 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
-package base
+package markdown
 
 import (
 	"bytes"
 	"fmt"
 	"io"
-	"net/http"
 	"path"
 	"path/filepath"
 	"regexp"
 	"strings"
 
 	"github.com/Unknwon/com"
+	"github.com/microcosm-cc/bluemonday"
 	"github.com/russross/blackfriday"
 	"golang.org/x/net/html"
 
+	"github.com/gogits/gogs/modules/base"
 	"github.com/gogits/gogs/modules/setting"
 )
 
-// TODO: put this into 'markdown' module.
+var Sanitizer = bluemonday.UGCPolicy()
 
-func isletter(c byte) bool {
-	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
-}
+// BuildSanitizer initializes sanitizer with allowed attributes based on settings.
+// This function should only be called once during entire application lifecycle.
+func BuildSanitizer() {
+	// Normal markdown-stuff
+	Sanitizer.AllowAttrs("class").Matching(regexp.MustCompile(`[\p{L}\p{N}\s\-_',:\[\]!\./\\\(\)&]*`)).OnElements("code")
+
+	// Checkboxes
+	Sanitizer.AllowAttrs("type").Matching(regexp.MustCompile(`^checkbox$`)).OnElements("input")
+	Sanitizer.AllowAttrs("checked", "disabled").OnElements("input")
 
-func isalnum(c byte) bool {
-	return (c >= '0' && c <= '9') || isletter(c)
+	// Custom URL-Schemes
+	Sanitizer.AllowURLSchemes(setting.Markdown.CustomURLSchemes...)
 }
 
 var validLinksPattern = regexp.MustCompile(`^[a-z][\w-]+://`)
 
+// isLink reports whether link fits valid format.
 func isLink(link []byte) bool {
 	return validLinksPattern.Match(link)
 }
 
+// IsMarkdownFile reports whether name looks like a Markdown file
+// based on its extension.
 func IsMarkdownFile(name string) bool {
 	name = strings.ToLower(name)
 	switch filepath.Ext(name) {
@@ -46,57 +56,46 @@ func IsMarkdownFile(name string) bool {
 	return false
 }
 
-func IsTextFile(data []byte) (string, bool) {
-	contentType := http.DetectContentType(data)
-	if strings.Index(contentType, "text/") != -1 {
-		return contentType, true
-	}
-	return contentType, false
-}
-
-func IsImageFile(data []byte) (string, bool) {
-	contentType := http.DetectContentType(data)
-	if strings.Index(contentType, "image/") != -1 {
-		return contentType, true
-	}
-	return contentType, false
-}
-
-// IsReadmeFile returns true if given file name suppose to be a README file.
+// IsReadmeFile reports whether name looks like a README file
+// based on its extension.
 func IsReadmeFile(name string) bool {
 	name = strings.ToLower(name)
 	if len(name) < 6 {
 		return false
 	} else if len(name) == 6 {
-		if name == "readme" {
-			return true
-		}
-		return false
-	}
-	if name[:7] == "readme." {
-		return true
+		return name == "readme"
 	}
-	return false
+	return name[:7] == "readme."
 }
 
 var (
-	MentionPattern     = regexp.MustCompile(`(\s|^)@[0-9a-zA-Z_\.]+`)
-	commitPattern      = regexp.MustCompile(`(\s|^)https?.*commit/[0-9a-zA-Z]+(#+[0-9a-zA-Z-]*)?`)
-	issueFullPattern   = regexp.MustCompile(`(\s|^)https?.*issues/[0-9]+(#+[0-9a-zA-Z-]*)?`)
-	issueIndexPattern  = regexp.MustCompile(`( |^|\()#[0-9]+\b`)
-	sha1CurrentPattern = regexp.MustCompile(`\b[0-9a-f]{40}\b`)
+	// MentionPattern matches string that mentions someone, e.g. @Unknwon
+	MentionPattern = regexp.MustCompile(`(\s|^)@[0-9a-zA-Z_\.]+`)
+
+	// CommitPattern matches link to certain commit with or without trailing hash,
+	// e.g. https://try.gogs.io/gogs/gogs/commit/d8a994ef243349f321568f9e36d5c3f444b99cae#diff-2
+	CommitPattern = regexp.MustCompile(`(\s|^)https?.*commit/[0-9a-zA-Z]+(#+[0-9a-zA-Z-]*)?`)
+
+	// IssueFullPattern matches link to an issue with or without trailing hash,
+	// e.g. https://try.gogs.io/gogs/gogs/issues/4#issue-685
+	IssueFullPattern = regexp.MustCompile(`(\s|^)https?.*issues/[0-9]+(#+[0-9a-zA-Z-]*)?`)
+	// IssueIndexPattern matches string that references to an issue, e.g. #1287
+	IssueIndexPattern = regexp.MustCompile(`( |^|\()#[0-9]+\b`)
+
+	// Sha1CurrentPattern matches string that represents a commit SHA, e.g. d8a994ef243349f321568f9e36d5c3f444b99cae
+	Sha1CurrentPattern = regexp.MustCompile(`\b[0-9a-f]{40}\b`)
 )
 
-type CustomRender struct {
+// Renderer is a extended version of underlying render object.
+type Renderer struct {
 	blackfriday.Renderer
 	urlPrefix string
 }
 
-func (r *CustomRender) Link(out *bytes.Buffer, link []byte, title []byte, content []byte) {
+// Link defines how formal links should be processed to produce corresponding HTML elements.
+func (r *Renderer) Link(out *bytes.Buffer, link []byte, title []byte, content []byte) {
 	if len(link) > 0 && !isLink(link) {
-		if link[0] == '#' {
-			// link = append([]byte(options.urlPrefix), link...)
-		} else {
+		if link[0] != '#' {
 			link = []byte(path.Join(r.urlPrefix, string(link)))
 		}
 	}
@@ -104,14 +103,17 @@ func (r *CustomRender) Link(out *bytes.Buffer, link []byte, title []byte, conten
 	r.Renderer.Link(out, link, title, content)
 }
 
-func (r *CustomRender) AutoLink(out *bytes.Buffer, link []byte, kind int) {
-	if kind != 1 {
+// AutoLink defines how auto-detected links should be processed to produce corresponding HTML elements.
+// Reference for kind: https://github.com/russross/blackfriday/blob/master/markdown.go#L69-L76
+func (r *Renderer) AutoLink(out *bytes.Buffer, link []byte, kind int) {
+	if kind != blackfriday.LINK_TYPE_NORMAL {
 		r.Renderer.AutoLink(out, link, kind)
 		return
 	}
 
-	// This method could only possibly serve one link at a time, no need to find all.
-	m := commitPattern.Find(link)
+	// Since this method could only possibly serve one link at a time,
+	// we do not need to find all.
+	m := CommitPattern.Find(link)
 	if m != nil {
 		m = bytes.TrimSpace(m)
 		i := strings.Index(string(m), "commit/")
@@ -119,11 +121,11 @@ func (r *CustomRender) AutoLink(out *bytes.Buffer, link []byte, kind int) {
 		if j == -1 {
 			j = len(m)
 		}
-		out.WriteString(fmt.Sprintf(` <code><a href="%s">%s</a></code>`, m, ShortSha(string(m[i+7:j]))))
+		out.WriteString(fmt.Sprintf(` <code><a href="%s">%s</a></code>`, m, base.ShortSha(string(m[i+7:j]))))
 		return
 	}
 
-	m = issueFullPattern.Find(link)
+	m = IssueFullPattern.Find(link)
 	if m != nil {
 		m = bytes.TrimSpace(m)
 		i := strings.Index(string(m), "issues/")
@@ -131,14 +133,16 @@ func (r *CustomRender) AutoLink(out *bytes.Buffer, link []byte, kind int) {
 		if j == -1 {
 			j = len(m)
 		}
-		out.WriteString(fmt.Sprintf(` <a href="%s">#%s</a>`, m, ShortSha(string(m[i+7:j]))))
+		out.WriteString(fmt.Sprintf(` <a href="%s">#%s</a>`, m, base.ShortSha(string(m[i+7:j]))))
 		return
 	}
 
 	r.Renderer.AutoLink(out, link, kind)
 }
 
-func (options *CustomRender) ListItem(out *bytes.Buffer, text []byte, flags int) {
+// ListItem defines how list items should be processed to produce corresponding HTML elements.
+func (options *Renderer) ListItem(out *bytes.Buffer, text []byte, flags int) {
+	// Detect procedures to draw checkboxes.
 	switch {
 	case bytes.HasPrefix(text, []byte("[ ] ")):
 		text = append([]byte(`<input type="checkbox" disabled="" />`), text[3:]...)
@@ -148,6 +152,8 @@ func (options *CustomRender) ListItem(out *bytes.Buffer, text []byte, flags int)
 	options.Renderer.ListItem(out, text, flags)
 }
 
+// Note: this section is for purpose of increase performance and
+// reduce memory allocation at runtime since they are constant literals.
 var (
 	svgSuffix         = []byte(".svg")
 	svgSuffixWithMark = []byte(".svg?")
@@ -155,11 +161,13 @@ var (
 	spaceEncodedBytes = []byte("%20")
 )
 
-func (r *CustomRender) Image(out *bytes.Buffer, link []byte, title []byte, alt []byte) {
+// Image defines how images should be processed to produce corresponding HTML elements.
+func (r *Renderer) Image(out *bytes.Buffer, link []byte, title []byte, alt []byte) {
 	prefix := strings.Replace(r.urlPrefix, "/src/", "/raw/", 1)
 	if len(link) > 0 {
 		if isLink(link) {
 			// External link with .svg suffix usually means CI status.
+			// TODO: define a keyword to allow non-svg images render as external link.
 			if bytes.HasSuffix(link, svgSuffix) || bytes.Contains(link, svgSuffixWithMark) {
 				r.Renderer.Image(out, link, title, alt)
 				return
@@ -180,6 +188,8 @@ func (r *CustomRender) Image(out *bytes.Buffer, link []byte, title []byte, alt [
 	out.WriteString("</a>")
 }
 
+// cutoutVerbosePrefix cutouts URL prefix including sub-path to
+// return a clean unified string of request URL path.
 func cutoutVerbosePrefix(prefix string) string {
 	count := 0
 	for i := 0; i < len(prefix); i++ {
@@ -193,29 +203,40 @@ func cutoutVerbosePrefix(prefix string) string {
 	return prefix
 }
 
+// RenderIssueIndexPattern renders issue indexes to corresponding links.
 func RenderIssueIndexPattern(rawBytes []byte, urlPrefix string, metas map[string]string) []byte {
 	urlPrefix = cutoutVerbosePrefix(urlPrefix)
-	ms := issueIndexPattern.FindAll(rawBytes, -1)
+	ms := IssueIndexPattern.FindAll(rawBytes, -1)
 	for _, m := range ms {
 		var space string
-		m2 := m
-		if m2[0] != '#' {
-			space = string(m2[0])
-			m2 = m2[1:]
+		if m[0] != '#' {
+			space = string(m[0])
+			m = m[1:]
 		}
 		if metas == nil {
 			rawBytes = bytes.Replace(rawBytes, m, []byte(fmt.Sprintf(`%s<a href="%s/issues/%s">%s</a>`,
-				space, urlPrefix, m2[1:], m2)), 1)
+				space, urlPrefix, m[1:], m)), 1)
 		} else {
 			// Support for external issue tracker
-			metas["index"] = string(m2[1:])
+			metas["index"] = string(m[1:])
 			rawBytes = bytes.Replace(rawBytes, m, []byte(fmt.Sprintf(`%s<a href="%s">%s</a>`,
-				space, com.Expand(metas["format"], metas), m2)), 1)
+				space, com.Expand(metas["format"], metas), m)), 1)
 		}
 	}
 	return rawBytes
 }
 
+// RenderSha1CurrentPattern renders SHA1 strings to corresponding links that assumes in the same repository.
+func RenderSha1CurrentPattern(rawBytes []byte, urlPrefix string) []byte {
+	ms := Sha1CurrentPattern.FindAll(rawBytes, -1)
+	for _, m := range ms {
+		rawBytes = bytes.Replace(rawBytes, m, []byte(fmt.Sprintf(
+			`<a href="%s/commit/%s"><code>%s</code></a>`, urlPrefix, m, base.ShortSha(string(m)))), -1)
+	}
+	return rawBytes
+}
+
+// RenderSpecialLink renders mentions, indexes and SHA1 strings to corresponding links.
 func RenderSpecialLink(rawBytes []byte, urlPrefix string, metas map[string]string) []byte {
 	ms := MentionPattern.FindAll(rawBytes, -1)
 	for _, m := range ms {
@@ -229,20 +250,12 @@ func RenderSpecialLink(rawBytes []byte, urlPrefix string, metas map[string]strin
 	return rawBytes
 }
 
-func RenderSha1CurrentPattern(rawBytes []byte, urlPrefix string) []byte {
-	ms := sha1CurrentPattern.FindAll(rawBytes, -1)
-	for _, m := range ms {
-		rawBytes = bytes.Replace(rawBytes, m, []byte(fmt.Sprintf(
-			`<a href="%s/commit/%s"><code>%s</code></a>`, urlPrefix, m, ShortSha(string(m)))), -1)
-	}
-	return rawBytes
-}
-
-func RenderRawMarkdown(body []byte, urlPrefix string) []byte {
+// RenderRaw renders Markdown to HTML without handling special links.
+func RenderRaw(body []byte, urlPrefix string) []byte {
 	htmlFlags := 0
 	htmlFlags |= blackfriday.HTML_SKIP_STYLE
 	htmlFlags |= blackfriday.HTML_OMIT_CONTENTS
-	renderer := &CustomRender{
+	renderer := &Renderer{
 		Renderer:  blackfriday.HtmlRenderer(htmlFlags, "", ""),
 		urlPrefix: urlPrefix,
 	}
@@ -272,9 +285,9 @@ var (
 
 var noEndTags = []string{"img", "input", "br", "hr"}
 
-// PostProcessMarkdown treats different types of HTML differently,
+// PostProcess treats different types of HTML differently,
 // and only renders special links for plain text blocks.
-func PostProcessMarkdown(rawHtml []byte, urlPrefix string, metas map[string]string) []byte {
+func PostProcess(rawHtml []byte, urlPrefix string, metas map[string]string) []byte {
 	startTags := make([]string, 0, 5)
 	var buf bytes.Buffer
 	tokenizer := html.NewTokenizer(bytes.NewReader(rawHtml))
@@ -342,13 +355,15 @@ OUTER_LOOP:
 	return rawHtml
 }
 
-func RenderMarkdown(rawBytes []byte, urlPrefix string, metas map[string]string) []byte {
-	result := RenderRawMarkdown(rawBytes, urlPrefix)
-	result = PostProcessMarkdown(result, urlPrefix, metas)
+// Render renders Markdown to HTML with special links.
+func Render(rawBytes []byte, urlPrefix string, metas map[string]string) []byte {
+	result := RenderRaw(rawBytes, urlPrefix)
+	result = PostProcess(result, urlPrefix, metas)
 	result = Sanitizer.SanitizeBytes(result)
 	return result
 }
 
-func RenderMarkdownString(raw, urlPrefix string, metas map[string]string) string {
-	return string(RenderMarkdown([]byte(raw), urlPrefix, metas))
+// RenderString renders Markdown to HTML with special links and returns string type.
+func RenderString(raw, urlPrefix string, metas map[string]string) string {
+	return string(Render([]byte(raw), urlPrefix, metas))
 }
diff --git a/modules/template/template.go b/modules/template/template.go
index aefb21946d..52ec054500 100644
--- a/modules/template/template.go
+++ b/modules/template/template.go
@@ -18,6 +18,7 @@ import (
 
 	"github.com/gogits/gogs/models"
 	"github.com/gogits/gogs/modules/base"
+	"github.com/gogits/gogs/modules/markdown"
 	"github.com/gogits/gogs/modules/setting"
 )
 
@@ -108,7 +109,7 @@ func Safe(raw string) template.HTML {
 }
 
 func Str2html(raw string) template.HTML {
-	return template.HTML(base.Sanitizer.Sanitize(raw))
+	return template.HTML(markdown.Sanitizer.Sanitize(raw))
 }
 
 func Range(l int) []int {
@@ -188,7 +189,7 @@ func ReplaceLeft(s, old, new string) string {
 // RenderCommitMessage renders commit message with XSS-safe and special links.
 func RenderCommitMessage(full bool, msg, urlPrefix string, metas map[string]string) template.HTML {
 	cleanMsg := template.HTMLEscapeString(msg)
-	fullMessage := string(base.RenderIssueIndexPattern([]byte(cleanMsg), urlPrefix, metas))
+	fullMessage := string(markdown.RenderIssueIndexPattern([]byte(cleanMsg), urlPrefix, metas))
 	msgLines := strings.Split(strings.TrimSpace(fullMessage), "\n")
 	numLines := len(msgLines)
 	if numLines == 0 {