diff options
author | Morgan Bazalgette <git@howl.moe> | 2018-02-27 08:09:18 +0100 |
---|---|---|
committer | Lauris BH <lauris@nix.lv> | 2018-02-27 09:09:18 +0200 |
commit | 535445c32ee730988033728b3b91c4d6f456e08c (patch) | |
tree | 34cd5b9807faf01018f47f74a34ed5b584df5158 | |
parent | 769ab1e4240f820efdb231832cb7957cb4902807 (diff) | |
download | gitea-535445c32ee730988033728b3b91c4d6f456e08c.tar.gz gitea-535445c32ee730988033728b3b91c4d6f456e08c.zip |
Rework special link parsing in the post-processing of markup (#3354)
* Get rid of autolink
* autolink in markdown
* Replace email addresses with mailto links
* better handling of links
* Remove autolink.js from footer
* Refactor entire html.go
* fix some bugs
* Make tests green, move what we can to html_internal_test, various other changes to processor logic
* Make markdown tests work again
This is just a description to allow me to force push in order to restart
the drone build.
* Fix failing markdown tests in routers/api/v1/misc
* Add license headers, log errors, future-proof <body>
* fix formatting
-rw-r--r-- | modules/markup/html.go | 894 | ||||
-rw-r--r-- | modules/markup/html_internal_test.go | 382 | ||||
-rw-r--r-- | modules/markup/html_test.go | 465 | ||||
-rw-r--r-- | modules/markup/markdown/markdown.go | 50 | ||||
-rw-r--r-- | modules/markup/markdown/markdown_test.go | 102 | ||||
-rw-r--r-- | modules/markup/markup.go | 8 | ||||
-rw-r--r-- | modules/templates/helper.go | 42 | ||||
-rw-r--r-- | public/js/index.js | 2 | ||||
-rw-r--r-- | public/vendor/plugins/autolink/LICENSE | 21 | ||||
-rw-r--r-- | public/vendor/plugins/autolink/autolink.js | 45 | ||||
-rw-r--r-- | routers/api/v1/misc/markdown_test.go | 4 | ||||
-rw-r--r-- | templates/base/footer.tmpl | 1 |
12 files changed, 1010 insertions, 1006 deletions
diff --git a/modules/markup/html.go b/modules/markup/html.go index e29cbe4457..53f57af1e0 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -6,8 +6,6 @@ package markup import ( "bytes" - "fmt" - "io" "net/url" "path" "path/filepath" @@ -20,6 +18,7 @@ import ( "github.com/Unknwon/com" "golang.org/x/net/html" + "golang.org/x/net/html/atom" ) // Issue name styles @@ -34,29 +33,40 @@ var ( // While fast, this is also incorrect and lead to false positives. // TODO: fix invalid linking issue - // MentionPattern matches string that mentions someone, e.g. @Unknwon - MentionPattern = regexp.MustCompile(`(\s|^|\W)@[0-9a-zA-Z-_\.]+`) + // mentionPattern matches all mentions in the form of "@user" + mentionPattern = regexp.MustCompile(`(?:\s|^|\W)(@[0-9a-zA-Z-_\.]+)`) - // IssueNumericPattern matches string that references to a numeric issue, e.g. #1287 - IssueNumericPattern = regexp.MustCompile(`( |^|\(|\[)#[0-9]+\b`) - // IssueAlphanumericPattern matches string that references to an alphanumeric issue, e.g. ABC-1234 - IssueAlphanumericPattern = regexp.MustCompile(`( |^|\(|\[)[A-Z]{1,10}-[1-9][0-9]*\b`) - // CrossReferenceIssueNumericPattern matches string that references a numeric issue in a different repository + // issueNumericPattern matches string that references to a numeric issue, e.g. #1287 + issueNumericPattern = regexp.MustCompile(`(?:\s|^|\W)(#[0-9]+)\b`) + // issueAlphanumericPattern matches string that references to an alphanumeric issue, e.g. ABC-1234 + issueAlphanumericPattern = regexp.MustCompile(`(?:\s|^|\W)([A-Z]{1,10}-[1-9][0-9]*)\b`) + // crossReferenceIssueNumericPattern matches string that references a numeric issue in a different repository // e.g. gogits/gogs#12345 - CrossReferenceIssueNumericPattern = regexp.MustCompile(`( |^)[0-9a-zA-Z-_\.]+/[0-9a-zA-Z-_\.]+#[0-9]+\b`) + crossReferenceIssueNumericPattern = regexp.MustCompile(`(?:\s|^|\W)([0-9a-zA-Z-_\.]+/[0-9a-zA-Z-_\.]+#[0-9]+)\b`) - // Sha1CurrentPattern matches string that represents a commit SHA, e.g. d8a994ef243349f321568f9e36d5c3f444b99cae + // sha1CurrentPattern matches string that represents a commit SHA, e.g. d8a994ef243349f321568f9e36d5c3f444b99cae // Although SHA1 hashes are 40 chars long, the regex matches the hash from 7 to 40 chars in length // so that abbreviated hash links can be used as well. This matches git and github useability. - Sha1CurrentPattern = regexp.MustCompile(`(?:^|\s|\()([0-9a-f]{7,40})\b`) + sha1CurrentPattern = regexp.MustCompile(`(?:\s|^|\W)([0-9a-f]{7,40})\b`) - // ShortLinkPattern matches short but difficult to parse [[name|link|arg=test]] syntax - ShortLinkPattern = regexp.MustCompile(`(\[\[.*?\]\]\w*)`) + // shortLinkPattern matches short but difficult to parse [[name|link|arg=test]] syntax + shortLinkPattern = regexp.MustCompile(`\[\[(.*?)\]\](\w*)`) - // AnySHA1Pattern allows to split url containing SHA into parts - AnySHA1Pattern = regexp.MustCompile(`(http\S*)://(\S+)/(\S+)/(\S+)/(\S+)/([0-9a-f]{40})(?:/?([^#\s]+)?(?:#(\S+))?)?`) + // anySHA1Pattern allows to split url containing SHA into parts + anySHA1Pattern = regexp.MustCompile(`https?://(?:\S+/){4}([0-9a-f]{40})/?([^#\s]+)?(?:#(\S+))?`) validLinksPattern = regexp.MustCompile(`^[a-z][\w-]+://`) + + // While this email regex is definitely not perfect and I'm sure you can come up + // with edge cases, it is still accepted by the CommonMark specification, as + // well as the HTML5 spec: + // http://spec.commonmark.org/0.28/#email-address + // https://html.spec.whatwg.org/multipage/input.html#e-mail-state-(type%3Demail) + emailRegex = regexp.MustCompile("[a-zA-Z0-9.!#$%&'*+\\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*") + + // matches http/https links. used for autlinking those. partly modified from + // the original present in autolink.js + linkRegex = regexp.MustCompile(`(?:(?:http|https):\/\/(?:[\-;:&=\+\$,\w]+@)?[A-Za-z0-9\.\-]+|(?:www\.|[\-;:&=\+\$,\w]+@)[A-Za-z0-9\.\-]+)(?:(?:\/[\+~%\/\.\w\-]*)?\??(?:[\-\+:=&;%@\.\w]*)#?(?:[\.\!\/\\\w]*))?`) ) // regexp for full links to issues/pulls @@ -72,6 +82,10 @@ func isLink(link []byte) bool { return validLinksPattern.Match(link) } +func isLinkStr(link string) bool { + return validLinksPattern.MatchString(link) +} + func getIssueFullPattern() *regexp.Regexp { if issueFullPattern == nil { appURL := setting.AppURL @@ -87,11 +101,12 @@ func getIssueFullPattern() *regexp.Regexp { // FindAllMentions matches mention patterns in given content // and returns a list of found user names without @ prefix. func FindAllMentions(content string) []string { - mentions := MentionPattern.FindAllString(content, -1) - for i := range mentions { - mentions[i] = mentions[i][strings.Index(mentions[i], "@")+1:] // Strip @ character + mentions := mentionPattern.FindAllStringSubmatch(content, -1) + ret := make([]string, len(mentions)) + for i, val := range mentions { + ret[i] = val[1][1:] } - return mentions + return ret } // cutoutVerbosePrefix cutouts URL prefix including sub-path to @@ -112,84 +127,6 @@ func cutoutVerbosePrefix(prefix string) string { return prefix } -// RenderIssueIndexPatternOptions options for RenderIssueIndexPattern function -type RenderIssueIndexPatternOptions struct { - // url to which non-special formatting should be linked. If empty, - // no such links will be added - DefaultURL string - URLPrefix string - Metas map[string]string -} - -// addText add text to the given buffer, adding a link to the default url -// if appropriate -func (opts RenderIssueIndexPatternOptions) addText(text []byte, buf *bytes.Buffer) { - if len(text) == 0 { - return - } else if len(opts.DefaultURL) == 0 { - buf.Write(text) - return - } - buf.WriteString(`<a rel="nofollow" href="`) - buf.WriteString(opts.DefaultURL) - buf.WriteString(`">`) - buf.Write(text) - buf.WriteString(`</a>`) -} - -// RenderIssueIndexPattern renders issue indexes to corresponding links. -func RenderIssueIndexPattern(rawBytes []byte, opts RenderIssueIndexPatternOptions) []byte { - opts.URLPrefix = cutoutVerbosePrefix(opts.URLPrefix) - - pattern := IssueNumericPattern - if opts.Metas["style"] == IssueNameStyleAlphanumeric { - pattern = IssueAlphanumericPattern - } - - var buf bytes.Buffer - remainder := rawBytes - for { - indices := pattern.FindIndex(remainder) - if indices == nil || len(indices) < 2 { - opts.addText(remainder, &buf) - return buf.Bytes() - } - startIndex := indices[0] - endIndex := indices[1] - opts.addText(remainder[:startIndex], &buf) - if remainder[startIndex] == '(' || remainder[startIndex] == ' ' { - buf.WriteByte(remainder[startIndex]) - startIndex++ - } - if opts.Metas == nil { - buf.WriteString(`<a href="`) - buf.WriteString(util.URLJoin( - opts.URLPrefix, "issues", string(remainder[startIndex+1:endIndex]))) - buf.WriteString(`">`) - buf.Write(remainder[startIndex:endIndex]) - buf.WriteString(`</a>`) - } else { - // Support for external issue tracker - buf.WriteString(`<a href="`) - if opts.Metas["style"] == IssueNameStyleAlphanumeric { - opts.Metas["index"] = string(remainder[startIndex:endIndex]) - } else { - opts.Metas["index"] = string(remainder[startIndex+1 : endIndex]) - } - buf.WriteString(com.Expand(opts.Metas["format"], opts.Metas)) - buf.WriteString(`">`) - buf.Write(remainder[startIndex:endIndex]) - buf.WriteString(`</a>`) - } - if endIndex < len(remainder) && - (remainder[endIndex] == ')' || remainder[endIndex] == ' ') { - buf.WriteByte(remainder[endIndex]) - endIndex++ - } - remainder = remainder[endIndex:] - } -} - // IsSameDomain checks if given url string has the same hostname as current Gitea instance func IsSameDomain(s string) bool { if strings.HasPrefix(s, "/") { @@ -204,350 +141,523 @@ func IsSameDomain(s string) bool { return false } -// renderFullSha1Pattern renders SHA containing URLs -func renderFullSha1Pattern(rawBytes []byte, urlPrefix string) []byte { - ms := AnySHA1Pattern.FindAllSubmatch(rawBytes, -1) - for _, m := range ms { - all := m[0] - protocol := string(m[1]) - paths := string(m[2]) - path := protocol + "://" + paths - author := string(m[3]) - repoName := string(m[4]) - path = util.URLJoin(path, author, repoName) - ltype := "src" - itemType := m[5] - if IsSameDomain(paths) { - ltype = string(itemType) - } else if string(itemType) == "commit" { - ltype = "commit" +type postProcessError struct { + context string + err error +} + +func (p *postProcessError) Error() string { + return "PostProcess: " + p.context + ", " + p.Error() +} + +type processor func(ctx *postProcessCtx, node *html.Node) + +var defaultProcessors = []processor{ + mentionProcessor, + shortLinkProcessor, + fullIssuePatternProcessor, + issueIndexPatternProcessor, + crossReferenceIssueIndexPatternProcessor, + fullSha1PatternProcessor, + sha1CurrentPatternProcessor, + emailAddressProcessor, + linkProcessor, +} + +type postProcessCtx struct { + metas map[string]string + urlPrefix string + isWikiMarkdown bool + + // processors used by this context. + procs []processor + + // if set to true, when an <a> is found, instead of just returning during + // visitNode, it will recursively visit the node exclusively running + // shortLinkProcessorFull with true. + visitLinksForShortLinks bool +} + +// PostProcess does the final required transformations to the passed raw HTML +// data, and ensures its validity. Transformations include: replacing links and +// emails with HTML links, parsing shortlinks in the format of [[Link]], like +// MediaWiki, linking issues in the format #ID, and mentions in the format +// @user, and others. +func PostProcess( + rawHTML []byte, + urlPrefix string, + metas map[string]string, + isWikiMarkdown bool, +) ([]byte, error) { + // create the context from the parameters + ctx := &postProcessCtx{ + metas: metas, + urlPrefix: urlPrefix, + isWikiMarkdown: isWikiMarkdown, + procs: defaultProcessors, + visitLinksForShortLinks: true, + } + return ctx.postProcess(rawHTML) +} + +var commitMessageProcessors = []processor{ + mentionProcessor, + fullIssuePatternProcessor, + issueIndexPatternProcessor, + crossReferenceIssueIndexPatternProcessor, + fullSha1PatternProcessor, + sha1CurrentPatternProcessor, + emailAddressProcessor, + linkProcessor, +} + +// RenderCommitMessage will use the same logic as PostProcess, but will disable +// the shortLinkProcessor and will add a defaultLinkProcessor if defaultLink is +// set, which changes every text node into a link to the passed default link. +func RenderCommitMessage( + rawHTML []byte, + urlPrefix, defaultLink string, + metas map[string]string, +) ([]byte, error) { + ctx := &postProcessCtx{ + metas: metas, + urlPrefix: urlPrefix, + procs: commitMessageProcessors, + } + if defaultLink != "" { + // we don't have to fear data races, because being + // commitMessageProcessors of fixed len and cap, every time we append + // something to it the slice is realloc+copied, so append always + // generates the slice ex-novo. + ctx.procs = append(ctx.procs, genDefaultLinkProcessor(defaultLink)) + } + return ctx.postProcess(rawHTML) +} + +var byteBodyTag = []byte("<body>") +var byteBodyTagClosing = []byte("</body>") + +func (ctx *postProcessCtx) postProcess(rawHTML []byte) ([]byte, error) { + if ctx.procs == nil { + ctx.procs = defaultProcessors + } + + // give a generous extra 50 bytes + res := make([]byte, 0, len(rawHTML)+50) + res = append(res, byteBodyTag...) + res = append(res, rawHTML...) + res = append(res, byteBodyTagClosing...) + + // parse the HTML + nodes, err := html.ParseFragment(bytes.NewReader(res), nil) + if err != nil { + return nil, &postProcessError{"invalid HTML", err} + } + + for _, node := range nodes { + ctx.visitNode(node) + } + + // Create buffer in which the data will be placed again. We know that the + // length will be at least that of res; to spare a few alloc+copy, we + // reuse res, resetting its length to 0. + buf := bytes.NewBuffer(res[:0]) + // Render everything to buf. + for _, node := range nodes { + err = html.Render(buf, node) + if err != nil { + return nil, &postProcessError{"error rendering processed HTML", err} } - sha := m[6] - var subtree string - if len(m) > 7 && len(m[7]) > 0 { - subtree = string(m[7]) + } + + // remove initial parts - because Render creates a whole HTML page. + res = buf.Bytes() + res = res[bytes.Index(res, byteBodyTag)+len(byteBodyTag) : bytes.LastIndex(res, byteBodyTagClosing)] + + // Everything done successfully, return parsed data. + return res, nil +} + +func (ctx *postProcessCtx) visitNode(node *html.Node) { + // We ignore code, pre and already generated links. + switch node.Type { + case html.TextNode: + ctx.textNode(node) + case html.ElementNode: + if node.Data == "a" || node.Data == "code" || node.Data == "pre" { + if node.Data == "a" && ctx.visitLinksForShortLinks { + ctx.visitNodeForShortLinks(node) + } + return } - var line []byte - if len(m) > 8 && len(m[8]) > 0 { - line = m[8] + for n := node.FirstChild; n != nil; n = n.NextSibling { + ctx.visitNode(n) } - urlSuffix := "" - text := base.ShortSha(string(sha)) - if subtree != "" { - urlSuffix = "/" + subtree - text += urlSuffix + } + // ignore everything else +} + +func (ctx *postProcessCtx) visitNodeForShortLinks(node *html.Node) { + switch node.Type { + case html.TextNode: + shortLinkProcessorFull(ctx, node, true) + case html.ElementNode: + if node.Data == "code" || node.Data == "pre" { + return } - if line != nil { - value := string(line) - urlSuffix += "#" - urlSuffix += value - text += " (" - text += value - text += ")" + for n := node.FirstChild; n != nil; n = n.NextSibling { + ctx.visitNodeForShortLinks(n) } - rawBytes = bytes.Replace(rawBytes, all, []byte(fmt.Sprintf( - `<a href="%s">%s</a>`, util.URLJoin(path, ltype, string(sha))+urlSuffix, text)), -1) } - return rawBytes } -// RenderFullIssuePattern renders issues-like URLs -func RenderFullIssuePattern(rawBytes []byte) []byte { - ms := getIssueFullPattern().FindAllSubmatch(rawBytes, -1) - for _, m := range ms { - all := m[0] - id := string(m[1]) - text := "#" + id - // TODO if m[2] is not nil, then link is to a comment, - // and we should indicate that in the text somehow - rawBytes = bytes.Replace(rawBytes, all, []byte(fmt.Sprintf( - `<a href="%s">%s</a>`, string(all), text)), -1) - } - return rawBytes +// textNode runs the passed node through various processors, in order to handle +// all kinds of special links handled by the post-processing. +func (ctx *postProcessCtx) textNode(node *html.Node) { + for _, processor := range ctx.procs { + processor(ctx, node) + } } -func firstIndexOfByte(sl []byte, target byte) int { - for i := 0; i < len(sl); i++ { - if sl[i] == target { - return i - } +func createLink(href, content string) *html.Node { + textNode := &html.Node{ + Type: html.TextNode, + Data: content, + } + linkNode := &html.Node{ + FirstChild: textNode, + LastChild: textNode, + Type: html.ElementNode, + Data: "a", + DataAtom: atom.A, + Attr: []html.Attribute{ + {Key: "href", Val: href}, + }, } - return -1 + textNode.Parent = linkNode + return linkNode } -func lastIndexOfByte(sl []byte, target byte) int { - for i := len(sl) - 1; i >= 0; i-- { - if sl[i] == target { - return i - } +// replaceContent takes a text node, and in its content it replaces a section of +// it with the specified newNode. An example to visualize how this can work can +// be found here: https://play.golang.org/p/5zP8NnHZ03s +func replaceContent(node *html.Node, i, j int, newNode *html.Node) { + // get the data before and after the match + before := node.Data[:i] + after := node.Data[j:] + + // Replace in the current node the text, so that it is only what it is + // supposed to have. + node.Data = before + + // Get the current next sibling, before which we place the replaced data, + // and after that we place the new text node. + nextSibling := node.NextSibling + node.Parent.InsertBefore(newNode, nextSibling) + if after != "" { + node.Parent.InsertBefore(&html.Node{ + Type: html.TextNode, + Data: after, + }, nextSibling) } - return -1 } -// RenderShortLinks processes [[syntax]] -// -// noLink flag disables making link tags when set to true -// so this function just replaces the whole [[...]] with the content text -// -// isWikiMarkdown is a flag to choose linking url prefix -func RenderShortLinks(rawBytes []byte, urlPrefix string, noLink bool, isWikiMarkdown bool) []byte { - ms := ShortLinkPattern.FindAll(rawBytes, -1) - for _, m := range ms { - orig := bytes.TrimSpace(m) - m = orig[2:] - tailPos := lastIndexOfByte(m, ']') + 1 - tail := []byte{} - if tailPos < len(m) { - tail = m[tailPos:] - m = m[:tailPos-1] - } - m = m[:len(m)-2] - props := map[string]string{} - - // MediaWiki uses [[link|text]], while GitHub uses [[text|link]] - // It makes page handling terrible, but we prefer GitHub syntax - // And fall back to MediaWiki only when it is obvious from the look - // Of text and link contents - sl := bytes.Split(m, []byte("|")) - for _, v := range sl { - switch bytes.Count(v, []byte("=")) { - - // Piped args without = sign, these are mandatory arguments - case 0: - { - sv := string(v) - if props["name"] == "" { - if isLink(v) { - // If we clearly see it is a link, we save it so - - // But first we need to ensure, that if both mandatory args provided - // look like links, we stick to GitHub syntax - if props["link"] != "" { - props["name"] = props["link"] - } - - props["link"] = strings.TrimSpace(sv) - } else { - props["name"] = sv - } - } else { - props["link"] = strings.TrimSpace(sv) - } - } +func mentionProcessor(_ *postProcessCtx, node *html.Node) { + m := mentionPattern.FindStringSubmatchIndex(node.Data) + if m == nil { + return + } + // Replace the mention with a link to the specified user. + mention := node.Data[m[2]:m[3]] + replaceContent(node, m[2], m[3], createLink(util.URLJoin(setting.AppURL, mention[1:]), mention)) +} + +func shortLinkProcessor(ctx *postProcessCtx, node *html.Node) { + shortLinkProcessorFull(ctx, node, false) +} + +func shortLinkProcessorFull(ctx *postProcessCtx, node *html.Node, noLink bool) { + m := shortLinkPattern.FindStringSubmatchIndex(node.Data) + if m == nil { + return + } - // Piped args with = sign, these are optional arguments - case 1: - { - sep := firstIndexOfByte(v, '=') - key, val := string(v[:sep]), html.UnescapeString(string(v[sep+1:])) - lastCharIndex := len(val) - 1 - if (val[0] == '"' || val[0] == '\'') && (val[lastCharIndex] == '"' || val[lastCharIndex] == '\'') { - val = val[1:lastCharIndex] + content := node.Data[m[2]:m[3]] + tail := node.Data[m[4]:m[5]] + props := make(map[string]string) + + // MediaWiki uses [[link|text]], while GitHub uses [[text|link]] + // It makes page handling terrible, but we prefer GitHub syntax + // And fall back to MediaWiki only when it is obvious from the look + // Of text and link contents + sl := strings.Split(content, "|") + for _, v := range sl { + if equalPos := strings.IndexByte(v, '='); equalPos == -1 { + // There is no equal in this argument; this is a mandatory arg + if props["name"] == "" { + if isLinkStr(v) { + // If we clearly see it is a link, we save it so + + // But first we need to ensure, that if both mandatory args provided + // look like links, we stick to GitHub syntax + if props["link"] != "" { + props["name"] = props["link"] } - props[key] = val + + props["link"] = strings.TrimSpace(v) + } else { + props["name"] = v } + } else { + props["link"] = strings.TrimSpace(v) + } + } else { + // There is an equal; optional argument. + + sep := strings.IndexByte(v, '=') + key, val := v[:sep], html.UnescapeString(v[sep+1:]) + + // When parsing HTML, x/net/html will change all quotes which are + // not used for syntax into UTF-8 quotes. So checking val[0] won't + // be enough, since that only checks a single byte. + if (strings.HasPrefix(val, "“") && strings.HasSuffix(val, "”")) || + (strings.HasPrefix(val, "‘") && strings.HasSuffix(val, "’")) { + const lenQuote = len("‘") + val = val[lenQuote : len(val)-lenQuote] } + props[key] = val } + } - var name string - var link string - if props["link"] != "" { - link = props["link"] - } else if props["name"] != "" { - link = props["name"] + var name, link string + if props["link"] != "" { + link = props["link"] + } else if props["name"] != "" { + link = props["name"] + } + if props["title"] != "" { + name = props["title"] + } else if props["name"] != "" { + name = props["name"] + } else { + name = link + } + + name += tail + image := false + switch ext := filepath.Ext(string(link)); ext { + // fast path: empty string, ignore + case "": + break + case ".jpg", ".jpeg", ".png", ".tif", ".tiff", ".webp", ".gif", ".bmp", ".ico", ".svg": + image = true + } + + childNode := &html.Node{} + linkNode := &html.Node{ + FirstChild: childNode, + LastChild: childNode, + Type: html.ElementNode, + Data: "a", + DataAtom: atom.A, + } + childNode.Parent = linkNode + absoluteLink := isLinkStr(link) + if !absoluteLink { + link = strings.Replace(link, " ", "+", -1) + } + urlPrefix := ctx.urlPrefix + if image { + if !absoluteLink { + if IsSameDomain(urlPrefix) { + urlPrefix = strings.Replace(urlPrefix, "/src/", "/raw/", 1) + } + if ctx.isWikiMarkdown { + link = util.URLJoin("wiki", "raw", link) + } + link = util.URLJoin(urlPrefix, link) } - if props["title"] != "" { - name = props["title"] - } else if props["name"] != "" { - name = props["name"] - } else { - name = link + title := props["title"] + if title == "" { + title = props["alt"] + } + if title == "" { + title = path.Base(string(name)) + } + alt := props["alt"] + if alt == "" { + alt = name } - name += string(tail) - image := false - ext := filepath.Ext(string(link)) - if ext != "" { - switch ext { - case ".jpg", ".jpeg", ".png", ".tif", ".tiff", ".webp", ".gif", ".bmp", ".ico", ".svg": - { - image = true - } - } + // make the childNode an image - if we can, we also place the alt + childNode.Type = html.ElementNode + childNode.Data = "img" + childNode.DataAtom = atom.Img + childNode.Attr = []html.Attribute{ + {Key: "src", Val: link}, + {Key: "title", Val: title}, + {Key: "alt", Val: alt}, } - absoluteLink := isLink([]byte(link)) - if !absoluteLink { - link = strings.Replace(link, " ", "+", -1) + if alt == "" { + childNode.Attr = childNode.Attr[:2] } - if image { - if !absoluteLink { - if IsSameDomain(urlPrefix) { - urlPrefix = strings.Replace(urlPrefix, "/src/", "/raw/", 1) - } - if isWikiMarkdown { - link = util.URLJoin("wiki", "raw", link) - } - link = util.URLJoin(urlPrefix, link) - } - title := props["title"] - if title == "" { - title = props["alt"] - } - if title == "" { - title = path.Base(string(name)) - } - alt := props["alt"] - if alt == "" { - alt = name - } - if alt != "" { - alt = `alt="` + alt + `"` - } - name = fmt.Sprintf(`<img src="%s" %s title="%s" />`, link, alt, title) - } else if !absoluteLink { - if isWikiMarkdown { + } else { + if !absoluteLink { + if ctx.isWikiMarkdown { link = util.URLJoin("wiki", link) } link = util.URLJoin(urlPrefix, link) } - if noLink { - rawBytes = bytes.Replace(rawBytes, orig, []byte(name), -1) - } else { - rawBytes = bytes.Replace(rawBytes, orig, - []byte(fmt.Sprintf(`<a href="%s">%s</a>`, link, name)), -1) - } + childNode.Type = html.TextNode + childNode.Data = name } - return rawBytes + if noLink { + linkNode = childNode + } else { + linkNode.Attr = []html.Attribute{{Key: "href", Val: link}} + } + replaceContent(node, m[0], m[1], linkNode) } -// RenderCrossReferenceIssueIndexPattern renders issue indexes from other repositories to corresponding links. -func RenderCrossReferenceIssueIndexPattern(rawBytes []byte, urlPrefix string, metas map[string]string) []byte { - ms := CrossReferenceIssueNumericPattern.FindAll(rawBytes, -1) - for _, m := range ms { - if m[0] == ' ' || m[0] == '(' { - m = m[1:] // ignore leading space or opening parentheses - } +func fullIssuePatternProcessor(ctx *postProcessCtx, node *html.Node) { + m := getIssueFullPattern().FindStringSubmatchIndex(node.Data) + if m == nil { + return + } + link := node.Data[m[0]:m[1]] + id := "#" + node.Data[m[2]:m[3]] + // TODO if m[4]:m[5] is not nil, then link is to a comment, + // and we should indicate that in the text somehow + replaceContent(node, m[0], m[1], createLink(link, id)) +} - repo := string(bytes.Split(m, []byte("#"))[0]) - issue := string(bytes.Split(m, []byte("#"))[1]) +func issueIndexPatternProcessor(ctx *postProcessCtx, node *html.Node) { + prefix := cutoutVerbosePrefix(ctx.urlPrefix) - link := fmt.Sprintf(`<a href="%s">%s</a>`, util.URLJoin(setting.AppURL, repo, "issues", issue), m) - rawBytes = bytes.Replace(rawBytes, m, []byte(link), 1) + // default to numeric pattern, unless alphanumeric is requested. + pattern := issueNumericPattern + if ctx.metas["style"] == IssueNameStyleAlphanumeric { + pattern = issueAlphanumericPattern } - return rawBytes -} -// renderSha1CurrentPattern renders SHA1 strings to corresponding links that assumes in the same repository. -func renderSha1CurrentPattern(rawBytes []byte, urlPrefix string) []byte { - ms := Sha1CurrentPattern.FindAllSubmatch(rawBytes, -1) - for _, m := range ms { - hash := m[1] - // The regex does not lie, it matches the hash pattern. - // However, a regex cannot know if a hash actually exists or not. - // We could assume that a SHA1 hash should probably contain alphas AND numerics - // but that is not always the case. - // Although unlikely, deadbeef and 1234567 are valid short forms of SHA1 hash - // as used by git and github for linking and thus we have to do similar. - rawBytes = bytes.Replace(rawBytes, hash, []byte(fmt.Sprintf( - `<a href="%s">%s</a>`, util.URLJoin(urlPrefix, "commit", string(hash)), base.ShortSha(string(hash)))), -1) - } - return rawBytes + match := pattern.FindStringSubmatchIndex(node.Data) + if match == nil { + return + } + id := node.Data[match[2]:match[3]] + var link *html.Node + if ctx.metas == nil { + link = createLink(util.URLJoin(prefix, "issues", id[1:]), id) + } else { + // Support for external issue tracker + if ctx.metas["style"] == IssueNameStyleAlphanumeric { + ctx.metas["index"] = id + } else { + ctx.metas["index"] = id[1:] + } + link = createLink(com.Expand(ctx.metas["format"], ctx.metas), id) + } + replaceContent(node, match[2], match[3], link) } -// RenderSpecialLink renders mentions, indexes and SHA1 strings to corresponding links. -func RenderSpecialLink(rawBytes []byte, urlPrefix string, metas map[string]string, isWikiMarkdown bool) []byte { - ms := MentionPattern.FindAll(rawBytes, -1) - for _, m := range ms { - m = m[bytes.Index(m, []byte("@")):] - rawBytes = bytes.Replace(rawBytes, m, - []byte(fmt.Sprintf(`<a href="%s">%s</a>`, util.URLJoin(setting.AppURL, string(m[1:])), m)), -1) - } - - rawBytes = RenderFullIssuePattern(rawBytes) - rawBytes = RenderShortLinks(rawBytes, urlPrefix, false, isWikiMarkdown) - rawBytes = RenderIssueIndexPattern(rawBytes, RenderIssueIndexPatternOptions{ - URLPrefix: urlPrefix, - Metas: metas, - }) - rawBytes = RenderCrossReferenceIssueIndexPattern(rawBytes, urlPrefix, metas) - rawBytes = renderFullSha1Pattern(rawBytes, urlPrefix) - rawBytes = renderSha1CurrentPattern(rawBytes, urlPrefix) - return rawBytes +func crossReferenceIssueIndexPatternProcessor(ctx *postProcessCtx, node *html.Node) { + m := crossReferenceIssueNumericPattern.FindStringSubmatchIndex(node.Data) + if m == nil { + return + } + ref := node.Data[m[2]:m[3]] + + parts := strings.SplitN(ref, "#", 2) + repo, issue := parts[0], parts[1] + + replaceContent(node, m[2], m[3], + createLink(util.URLJoin(setting.AppURL, repo, "issues", issue), ref)) } -var ( - leftAngleBracket = []byte("</") - rightAngleBracket = []byte(">") -) +// fullSha1PatternProcessor renders SHA containing URLs +func fullSha1PatternProcessor(ctx *postProcessCtx, node *html.Node) { + m := anySHA1Pattern.FindStringSubmatchIndex(node.Data) + if m == nil { + return + } + // take out what's relevant + urlFull := node.Data[m[0]:m[1]] + hash := node.Data[m[2]:m[3]] -var noEndTags = []string{"img", "input", "br", "hr"} - -// PostProcess treats different types of HTML differently, -// and only renders special links for plain text blocks. -func PostProcess(rawHTML []byte, urlPrefix string, metas map[string]string, isWikiMarkdown bool) []byte { - startTags := make([]string, 0, 5) - var buf bytes.Buffer - tokenizer := html.NewTokenizer(bytes.NewReader(rawHTML)) - -OUTER_LOOP: - for html.ErrorToken != tokenizer.Next() { - token := tokenizer.Token() - switch token.Type { - case html.TextToken: - buf.Write(RenderSpecialLink([]byte(token.String()), urlPrefix, metas, isWikiMarkdown)) - - case html.StartTagToken: - buf.WriteString(token.String()) - tagName := token.Data - // If this is an excluded tag, we skip processing all output until a close tag is encountered. - if strings.EqualFold("a", tagName) || strings.EqualFold("code", tagName) || strings.EqualFold("pre", tagName) { - stackNum := 1 - for html.ErrorToken != tokenizer.Next() { - token = tokenizer.Token() - - // Copy the token to the output verbatim - buf.Write(RenderShortLinks([]byte(token.String()), urlPrefix, true, isWikiMarkdown)) - - if token.Type == html.StartTagToken && !com.IsSliceContainsStr(noEndTags, token.Data) { - stackNum++ - } + var subtree, line string - // If this is the close tag to the outer-most, we are done - if token.Type == html.EndTagToken { - stackNum-- + // optional, we do them depending on the length. + if m[7] > 0 { + line = node.Data[m[6]:m[7]] + } + if m[5] > 0 { + subtree = node.Data[m[4]:m[5]] + } - if stackNum <= 0 && strings.EqualFold(tagName, token.Data) { - break - } - } - } - continue OUTER_LOOP - } + text := base.ShortSha(hash) + if subtree != "" { + text += "/" + subtree + } + if line != "" { + text += " (" + text += line + text += ")" + } - if !com.IsSliceContainsStr(noEndTags, tagName) { - startTags = append(startTags, tagName) - } + replaceContent(node, m[0], m[1], createLink(urlFull, text)) +} - case html.EndTagToken: - if len(startTags) == 0 { - buf.WriteString(token.String()) - break - } +// sha1CurrentPatternProcessor renders SHA1 strings to corresponding links that +// are assumed to be in the same repository. +func sha1CurrentPatternProcessor(ctx *postProcessCtx, node *html.Node) { + m := sha1CurrentPattern.FindStringSubmatchIndex(node.Data) + if m == nil { + return + } + hash := node.Data[m[2]:m[3]] + // The regex does not lie, it matches the hash pattern. + // However, a regex cannot know if a hash actually exists or not. + // We could assume that a SHA1 hash should probably contain alphas AND numerics + // but that is not always the case. + // Although unlikely, deadbeef and 1234567 are valid short forms of SHA1 hash + // as used by git and github for linking and thus we have to do similar. + replaceContent(node, m[2], m[3], + createLink(util.URLJoin(ctx.urlPrefix, "commit", hash), base.ShortSha(hash))) +} - buf.Write(leftAngleBracket) - buf.WriteString(startTags[len(startTags)-1]) - buf.Write(rightAngleBracket) - startTags = startTags[:len(startTags)-1] - default: - buf.WriteString(token.String()) - } +// emailAddressProcessor replaces raw email addresses with a mailto: link. +func emailAddressProcessor(ctx *postProcessCtx, node *html.Node) { + m := emailRegex.FindStringIndex(node.Data) + if m == nil { + return } + mail := node.Data[m[0]:m[1]] + replaceContent(node, m[0], m[1], createLink("mailto:"+mail, mail)) +} - if io.EOF == tokenizer.Err() { - return buf.Bytes() +// linkProcessor creates links for any HTTP or HTTPS URL not captured by +// markdown. +func linkProcessor(ctx *postProcessCtx, node *html.Node) { + m := linkRegex.FindStringIndex(node.Data) + if m == nil { + return } + uri := node.Data[m[0]:m[1]] + replaceContent(node, m[0], m[1], createLink(uri, uri)) +} + +func genDefaultLinkProcessor(defaultLink string) processor { + return func(ctx *postProcessCtx, node *html.Node) { + ch := &html.Node{ + Parent: node, + Type: html.TextNode, + Data: node.Data, + } - // If we are not at the end of the input, then some other parsing error has occurred, - // so return the input verbatim. - return rawHTML + node.Type = html.ElementNode + node.Data = "a" + node.DataAtom = atom.A + node.Attr = []html.Attribute{{Key: "href", Val: defaultLink}} + node.FirstChild, node.LastChild = ch, ch + } } diff --git a/modules/markup/html_internal_test.go b/modules/markup/html_internal_test.go new file mode 100644 index 0000000000..ff07bab913 --- /dev/null +++ b/modules/markup/html_internal_test.go @@ -0,0 +1,382 @@ +// Copyright 2018 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +package markup + +import ( + "fmt" + "strconv" + "strings" + "testing" + + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" + + "github.com/stretchr/testify/assert" +) + +const AppURL = "http://localhost:3000/" +const Repo = "gogits/gogs" +const AppSubURL = AppURL + Repo + "/" + +// alphanumLink an HTML link to an alphanumeric-style issue +func alphanumIssueLink(baseURL string, name string) string { + return link(util.URLJoin(baseURL, name), name) +} + +// numericLink an HTML to a numeric-style issue +func numericIssueLink(baseURL string, index int) string { + return link(util.URLJoin(baseURL, strconv.Itoa(index)), fmt.Sprintf("#%d", index)) +} + +// urlContentsLink an HTML link whose contents is the target URL +func urlContentsLink(href string) string { + return link(href, href) +} + +// link an HTML link +func link(href, contents string) string { + return fmt.Sprintf("<a href=\"%s\">%s</a>", href, contents) +} + +var numericMetas = map[string]string{ + "format": "https://someurl.com/{user}/{repo}/{index}", + "user": "someUser", + "repo": "someRepo", + "style": IssueNameStyleNumeric, +} + +var alphanumericMetas = map[string]string{ + "format": "https://someurl.com/{user}/{repo}/{index}", + "user": "someUser", + "repo": "someRepo", + "style": IssueNameStyleAlphanumeric, +} + +func TestRender_IssueIndexPattern(t *testing.T) { + // numeric: render inputs without valid mentions + test := func(s string) { + testRenderIssueIndexPattern(t, s, s, nil) + testRenderIssueIndexPattern(t, s, s, &postProcessCtx{metas: numericMetas}) + } + + // should not render anything when there are no mentions + test("") + test("this is a test") + test("test 123 123 1234") + test("#") + test("# # #") + test("# 123") + test("#abcd") + test("test#1234") + test("#1234test") + test(" test #1234test") + + // should not render issue mention without leading space + test("test#54321 issue") + + // should not render issue mention without trailing space + test("test #54321issue") +} + +func TestRender_IssueIndexPattern2(t *testing.T) { + setting.AppURL = AppURL + setting.AppSubURL = AppSubURL + + // numeric: render inputs with valid mentions + test := func(s, expectedFmt string, indices ...int) { + links := make([]interface{}, len(indices)) + for i, index := range indices { + links[i] = numericIssueLink(util.URLJoin(setting.AppSubURL, "issues"), index) + } + expectedNil := fmt.Sprintf(expectedFmt, links...) + testRenderIssueIndexPattern(t, s, expectedNil, nil) + + for i, index := range indices { + links[i] = numericIssueLink("https://someurl.com/someUser/someRepo/", index) + } + expectedNum := fmt.Sprintf(expectedFmt, links...) + testRenderIssueIndexPattern(t, s, expectedNum, &postProcessCtx{metas: numericMetas}) + } + + // should render freestanding mentions + test("#1234 test", "%s test", 1234) + test("test #8 issue", "test %s issue", 8) + test("test issue #1234", "test issue %s", 1234) + + // should render mentions in parentheses + test("(#54321 issue)", "(%s issue)", 54321) + test("test (#9801 extra) issue", "test (%s extra) issue", 9801) + test("test (#1)", "test (%s)", 1) + + // should render multiple issue mentions in the same line + test("#54321 #1243", "%s %s", 54321, 1243) + test("wow (#54321 #1243)", "wow (%s %s)", 54321, 1243) + test("(#4)(#5)", "(%s)(%s)", 4, 5) + test("#1 (#4321) test", "%s (%s) test", 1, 4321) +} + +func TestRender_IssueIndexPattern3(t *testing.T) { + setting.AppURL = AppURL + setting.AppSubURL = AppSubURL + + // alphanumeric: render inputs without valid mentions + test := func(s string) { + testRenderIssueIndexPattern(t, s, s, &postProcessCtx{metas: alphanumericMetas}) + } + test("") + test("this is a test") + test("test 123 123 1234") + test("#") + test("# 123") + test("#abcd") + test("test #123") + test("abc-1234") // issue prefix must be capital + test("ABc-1234") // issue prefix must be _all_ capital + test("ABCDEFGHIJK-1234") // the limit is 10 characters in the prefix + test("ABC1234") // dash is required + test("test ABC- test") // number is required + test("test -1234 test") // prefix is required + test("testABC-123 test") // leading space is required + test("test ABC-123test") // trailing space is required + test("ABC-0123") // no leading zero +} + +func TestRender_IssueIndexPattern4(t *testing.T) { + setting.AppURL = AppURL + setting.AppSubURL = AppSubURL + + // alphanumeric: render inputs with valid mentions + test := func(s, expectedFmt string, names ...string) { + links := make([]interface{}, len(names)) + for i, name := range names { + links[i] = alphanumIssueLink("https://someurl.com/someUser/someRepo/", name) + } + expected := fmt.Sprintf(expectedFmt, links...) + testRenderIssueIndexPattern(t, s, expected, &postProcessCtx{metas: alphanumericMetas}) + } + test("OTT-1234 test", "%s test", "OTT-1234") + test("test T-12 issue", "test %s issue", "T-12") + test("test issue ABCDEFGHIJ-1234567890", "test issue %s", "ABCDEFGHIJ-1234567890") +} + +func testRenderIssueIndexPattern(t *testing.T, input, expected string, ctx *postProcessCtx) { + if ctx == nil { + ctx = new(postProcessCtx) + } + ctx.procs = []processor{issueIndexPatternProcessor} + if ctx.urlPrefix == "" { + ctx.urlPrefix = AppSubURL + } + res, err := ctx.postProcess([]byte(input)) + assert.NoError(t, err) + assert.Equal(t, expected, string(res)) +} + +func TestRender_AutoLink(t *testing.T) { + setting.AppURL = AppURL + setting.AppSubURL = AppSubURL + + test := func(input, expected string) { + buffer, err := PostProcess([]byte(input), setting.AppSubURL, nil, false) + assert.Equal(t, err, nil) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer))) + buffer, err = PostProcess([]byte(input), setting.AppSubURL, nil, true) + assert.Equal(t, err, nil) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer))) + } + + // render valid issue URLs + test(util.URLJoin(setting.AppSubURL, "issues", "3333"), + numericIssueLink(util.URLJoin(setting.AppSubURL, "issues"), 3333)) + + // render valid commit URLs + tmp := util.URLJoin(AppSubURL, "commit", "d8a994ef243349f321568f9e36d5c3f444b99cae") + test(tmp, "<a href=\""+tmp+"\">d8a994ef24</a>") + tmp += "#diff-2" + test(tmp, "<a href=\""+tmp+"\">d8a994ef24 (diff-2)</a>") + + // render other commit URLs + tmp = "https://external-link.gogs.io/gogs/gogs/commit/d8a994ef243349f321568f9e36d5c3f444b99cae#diff-2" + test(tmp, "<a href=\""+tmp+"\">d8a994ef24 (diff-2)</a>") +} + +func TestRender_FullIssueURLs(t *testing.T) { + setting.AppURL = AppURL + setting.AppSubURL = AppSubURL + + test := func(input, expected string) { + ctx := new(postProcessCtx) + ctx.procs = []processor{fullIssuePatternProcessor} + if ctx.urlPrefix == "" { + ctx.urlPrefix = AppSubURL + } + result, err := ctx.postProcess([]byte(input)) + assert.NoError(t, err) + assert.Equal(t, expected, string(result)) + } + test("Here is a link https://git.osgeo.org/gogs/postgis/postgis/pulls/6", + "Here is a link https://git.osgeo.org/gogs/postgis/postgis/pulls/6") + test("Look here http://localhost:3000/person/repo/issues/4", + `Look here <a href="http://localhost:3000/person/repo/issues/4">#4</a>`) + test("http://localhost:3000/person/repo/issues/4#issuecomment-1234", + `<a href="http://localhost:3000/person/repo/issues/4#issuecomment-1234">#4</a>`) +} + +func TestRegExp_issueNumericPattern(t *testing.T) { + trueTestCases := []string{ + "#1234", + "#0", + "#1234567890987654321", + " #12", + } + falseTestCases := []string{ + "# 1234", + "# 0", + "# ", + "#", + "#ABC", + "#1A2B", + "", + "ABC", + } + + for _, testCase := range trueTestCases { + assert.True(t, issueNumericPattern.MatchString(testCase)) + } + for _, testCase := range falseTestCases { + assert.False(t, issueNumericPattern.MatchString(testCase)) + } +} + +func TestRegExp_sha1CurrentPattern(t *testing.T) { + trueTestCases := []string{ + "d8a994ef243349f321568f9e36d5c3f444b99cae", + "abcdefabcdefabcdefabcdefabcdefabcdefabcd", + } + falseTestCases := []string{ + "test", + "abcdefg", + "abcdefghijklmnopqrstuvwxyzabcdefghijklmn", + "abcdefghijklmnopqrstuvwxyzabcdefghijklmO", + } + + for _, testCase := range trueTestCases { + assert.True(t, sha1CurrentPattern.MatchString(testCase)) + } + for _, testCase := range falseTestCases { + assert.False(t, sha1CurrentPattern.MatchString(testCase)) + } +} + +func TestRegExp_anySHA1Pattern(t *testing.T) { + testCases := map[string][]string{ + "https://github.com/jquery/jquery/blob/a644101ed04d0beacea864ce805e0c4f86ba1cd1/test/unit/event.js#L2703": { + "a644101ed04d0beacea864ce805e0c4f86ba1cd1", + "test/unit/event.js", + "L2703", + }, + "https://github.com/jquery/jquery/blob/a644101ed04d0beacea864ce805e0c4f86ba1cd1/test/unit/event.js": { + "a644101ed04d0beacea864ce805e0c4f86ba1cd1", + "test/unit/event.js", + "", + }, + "https://github.com/jquery/jquery/commit/0705be475092aede1eddae01319ec931fb9c65fc": { + "0705be475092aede1eddae01319ec931fb9c65fc", + "", + "", + }, + "https://github.com/jquery/jquery/tree/0705be475092aede1eddae01319ec931fb9c65fc/src": { + "0705be475092aede1eddae01319ec931fb9c65fc", + "src", + "", + }, + "https://try.gogs.io/gogs/gogs/commit/d8a994ef243349f321568f9e36d5c3f444b99cae#diff-2": { + "d8a994ef243349f321568f9e36d5c3f444b99cae", + "", + "diff-2", + }, + } + + for k, v := range testCases { + assert.Equal(t, anySHA1Pattern.FindStringSubmatch(k)[1:], v) + } +} + +func TestRegExp_mentionPattern(t *testing.T) { + trueTestCases := []string{ + "@Unknwon", + "@ANT_123", + "@xxx-DiN0-z-A..uru..s-xxx", + " @lol ", + " @Te/st", + } + falseTestCases := []string{ + "@ 0", + "@ ", + "@", + "", + "ABC", + } + + for _, testCase := range trueTestCases { + res := mentionPattern.MatchString(testCase) + assert.True(t, res) + } + for _, testCase := range falseTestCases { + res := mentionPattern.MatchString(testCase) + assert.False(t, res) + } +} + +func TestRegExp_issueAlphanumericPattern(t *testing.T) { + trueTestCases := []string{ + "ABC-1234", + "A-1", + "RC-80", + "ABCDEFGHIJ-1234567890987654321234567890", + } + falseTestCases := []string{ + "RC-08", + "PR-0", + "ABCDEFGHIJK-1", + "PR_1", + "", + "#ABC", + "", + "ABC", + "GG-", + "rm-1", + } + + for _, testCase := range trueTestCases { + assert.True(t, issueAlphanumericPattern.MatchString(testCase)) + } + for _, testCase := range falseTestCases { + assert.False(t, issueAlphanumericPattern.MatchString(testCase)) + } +} + +func TestRegExp_shortLinkPattern(t *testing.T) { + trueTestCases := []string{ + "[[stuff]]", + "[[]]", + "[[stuff|title=Difficult name with spaces*!]]", + } + falseTestCases := []string{ + "test", + "abcdefg", + "[[]", + "[[", + "[]", + "]]", + "abcdefghijklmnopqrstuvwxyz", + } + + for _, testCase := range trueTestCases { + assert.True(t, shortLinkPattern.MatchString(testCase)) + } + for _, testCase := range falseTestCases { + assert.False(t, shortLinkPattern.MatchString(testCase)) + } +} diff --git a/modules/markup/html_test.go b/modules/markup/html_test.go index 92f815ad94..203db243dc 100644 --- a/modules/markup/html_test.go +++ b/modules/markup/html_test.go @@ -5,227 +5,17 @@ package markup_test import ( - "fmt" - "strconv" "strings" "testing" . "code.gitea.io/gitea/modules/markup" - _ "code.gitea.io/gitea/modules/markup/markdown" + "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" ) -const AppURL = "http://localhost:3000/" -const Repo = "gogits/gogs" -const AppSubURL = AppURL + Repo + "/" - -var numericMetas = map[string]string{ - "format": "https://someurl.com/{user}/{repo}/{index}", - "user": "someUser", - "repo": "someRepo", - "style": IssueNameStyleNumeric, -} - -var alphanumericMetas = map[string]string{ - "format": "https://someurl.com/{user}/{repo}/{index}", - "user": "someUser", - "repo": "someRepo", - "style": IssueNameStyleAlphanumeric, -} - -// numericLink an HTML to a numeric-style issue -func numericIssueLink(baseURL string, index int) string { - return link(util.URLJoin(baseURL, strconv.Itoa(index)), fmt.Sprintf("#%d", index)) -} - -// alphanumLink an HTML link to an alphanumeric-style issue -func alphanumIssueLink(baseURL string, name string) string { - return link(util.URLJoin(baseURL, name), name) -} - -// urlContentsLink an HTML link whose contents is the target URL -func urlContentsLink(href string) string { - return link(href, href) -} - -// link an HTML link -func link(href, contents string) string { - return fmt.Sprintf("<a href=\"%s\">%s</a>", href, contents) -} - -func testRenderIssueIndexPattern(t *testing.T, input, expected string, opts RenderIssueIndexPatternOptions) { - if len(opts.URLPrefix) == 0 { - opts.URLPrefix = AppSubURL - } - actual := string(RenderIssueIndexPattern([]byte(input), opts)) - assert.Equal(t, expected, actual) -} - -func TestRender_IssueIndexPattern(t *testing.T) { - // numeric: render inputs without valid mentions - test := func(s string) { - testRenderIssueIndexPattern(t, s, s, RenderIssueIndexPatternOptions{}) - testRenderIssueIndexPattern(t, s, s, RenderIssueIndexPatternOptions{Metas: numericMetas}) - } - - // should not render anything when there are no mentions - test("") - test("this is a test") - test("test 123 123 1234") - test("#") - test("# # #") - test("# 123") - test("#abcd") - test("##1234") - test("test#1234") - test("#1234test") - test(" test #1234test") - - // should not render issue mention without leading space - test("test#54321 issue") - - // should not render issue mention without trailing space - test("test #54321issue") -} - -func TestRender_IssueIndexPattern2(t *testing.T) { - setting.AppURL = AppURL - setting.AppSubURL = AppSubURL - - // numeric: render inputs with valid mentions - test := func(s, expectedFmt string, indices ...int) { - links := make([]interface{}, len(indices)) - for i, index := range indices { - links[i] = numericIssueLink(util.URLJoin(setting.AppSubURL, "issues"), index) - } - expectedNil := fmt.Sprintf(expectedFmt, links...) - testRenderIssueIndexPattern(t, s, expectedNil, RenderIssueIndexPatternOptions{}) - - for i, index := range indices { - links[i] = numericIssueLink("https://someurl.com/someUser/someRepo/", index) - } - expectedNum := fmt.Sprintf(expectedFmt, links...) - testRenderIssueIndexPattern(t, s, expectedNum, RenderIssueIndexPatternOptions{Metas: numericMetas}) - } - - // should render freestanding mentions - test("#1234 test", "%s test", 1234) - test("test #8 issue", "test %s issue", 8) - test("test issue #1234", "test issue %s", 1234) - - // should render mentions in parentheses - test("(#54321 issue)", "(%s issue)", 54321) - test("test (#9801 extra) issue", "test (%s extra) issue", 9801) - test("test (#1)", "test (%s)", 1) - - // should render multiple issue mentions in the same line - test("#54321 #1243", "%s %s", 54321, 1243) - test("wow (#54321 #1243)", "wow (%s %s)", 54321, 1243) - test("(#4)(#5)", "(%s)(%s)", 4, 5) - test("#1 (#4321) test", "%s (%s) test", 1, 4321) -} - -func TestRender_IssueIndexPattern3(t *testing.T) { - setting.AppURL = AppURL - setting.AppSubURL = AppSubURL - - // alphanumeric: render inputs without valid mentions - test := func(s string) { - testRenderIssueIndexPattern(t, s, s, RenderIssueIndexPatternOptions{Metas: alphanumericMetas}) - } - test("") - test("this is a test") - test("test 123 123 1234") - test("#") - test("##1234") - test("# 123") - test("#abcd") - test("test #123") - test("abc-1234") // issue prefix must be capital - test("ABc-1234") // issue prefix must be _all_ capital - test("ABCDEFGHIJK-1234") // the limit is 10 characters in the prefix - test("ABC1234") // dash is required - test("test ABC- test") // number is required - test("test -1234 test") // prefix is required - test("testABC-123 test") // leading space is required - test("test ABC-123test") // trailing space is required - test("ABC-0123") // no leading zero -} - -func TestRender_IssueIndexPattern4(t *testing.T) { - setting.AppURL = AppURL - setting.AppSubURL = AppSubURL - - // alphanumeric: render inputs with valid mentions - test := func(s, expectedFmt string, names ...string) { - links := make([]interface{}, len(names)) - for i, name := range names { - links[i] = alphanumIssueLink("https://someurl.com/someUser/someRepo/", name) - } - expected := fmt.Sprintf(expectedFmt, links...) - testRenderIssueIndexPattern(t, s, expected, RenderIssueIndexPatternOptions{Metas: alphanumericMetas}) - } - test("OTT-1234 test", "%s test", "OTT-1234") - test("test T-12 issue", "test %s issue", "T-12") - test("test issue ABCDEFGHIJ-1234567890", "test issue %s", "ABCDEFGHIJ-1234567890") -} - -func TestRenderIssueIndexPatternWithDefaultURL(t *testing.T) { - setting.AppURL = AppURL - setting.AppSubURL = AppSubURL - - test := func(input string, expected string) { - testRenderIssueIndexPattern(t, input, expected, RenderIssueIndexPatternOptions{ - DefaultURL: AppURL, - }) - } - test("hello #123 world", - fmt.Sprintf(`<a rel="nofollow" href="%s">hello</a> `, AppURL)+ - fmt.Sprintf(`<a href="%sissues/123">#123</a> `, AppSubURL)+ - fmt.Sprintf(`<a rel="nofollow" href="%s">world</a>`, AppURL)) - test("hello (#123) world", - fmt.Sprintf(`<a rel="nofollow" href="%s">hello </a>`, AppURL)+ - fmt.Sprintf(`(<a href="%sissues/123">#123</a>)`, AppSubURL)+ - fmt.Sprintf(`<a rel="nofollow" href="%s"> world</a>`, AppURL)) -} - -func TestRender_AutoLink(t *testing.T) { - setting.AppURL = AppURL - setting.AppSubURL = AppSubURL - - test := func(input, expected string) { - buffer := RenderSpecialLink([]byte(input), setting.AppSubURL, nil, false) - assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer))) - buffer = RenderSpecialLink([]byte(input), setting.AppSubURL, nil, true) - assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer))) - } - - // render valid issue URLs - test(util.URLJoin(setting.AppSubURL, "issues", "3333"), - numericIssueLink(util.URLJoin(setting.AppSubURL, "issues"), 3333)) - - // render external issue URLs - for _, externalURL := range []string{ - "http://1111/2222/ssss-issues/3333?param=blah&blahh=333", - "http://test.com/issues/33333", - "https://issues/333"} { - test(externalURL, externalURL) - } - - // render valid commit URLs - tmp := util.URLJoin(AppSubURL, "commit", "d8a994ef243349f321568f9e36d5c3f444b99cae") - test(tmp, "<a href=\""+tmp+"\">d8a994ef24</a>") - tmp += "#diff-2" - test(tmp, "<a href=\""+tmp+"\">d8a994ef24 (diff-2)</a>") - - // render other commit URLs - tmp = "https://external-link.gogs.io/gogs/gogs/commit/d8a994ef243349f321568f9e36d5c3f444b99cae#diff-2" - test(tmp, "<a href=\""+tmp+"\">d8a994ef24 (diff-2)</a>") -} - func TestRender_Commits(t *testing.T) { setting.AppURL = AppURL setting.AppSubURL = AppSubURL @@ -239,13 +29,12 @@ func TestRender_Commits(t *testing.T) { var commit = util.URLJoin(AppSubURL, "commit", sha) var subtree = util.URLJoin(commit, "src") var tree = strings.Replace(subtree, "/commit/", "/tree/", -1) - var src = strings.Replace(subtree, "/commit/", "/src/", -1) test(sha, `<p><a href="`+commit+`" rel="nofollow">b6dd6210ea</a></p>`) test(sha[:7], `<p><a href="`+commit[:len(commit)-(40-7)]+`" rel="nofollow">b6dd621</a></p>`) test(sha[:39], `<p><a href="`+commit[:len(commit)-(40-39)]+`" rel="nofollow">b6dd6210ea</a></p>`) test(commit, `<p><a href="`+commit+`" rel="nofollow">b6dd6210ea</a></p>`) - test(tree, `<p><a href="`+src+`" rel="nofollow">b6dd6210ea/src</a></p>`) + test(tree, `<p><a href="`+tree+`" rel="nofollow">b6dd6210ea/src</a></p>`) test("commit "+sha, `<p>commit <a href="`+commit+`" rel="nofollow">b6dd6210ea</a></p>`) } @@ -266,193 +55,6 @@ func TestRender_CrossReferences(t *testing.T) { `<p><a href="`+util.URLJoin(AppURL, "go-gitea", "gitea", "issues", "12345")+`" rel="nofollow">go-gitea/gitea#12345</a></p>`) } -func TestRender_FullIssueURLs(t *testing.T) { - setting.AppURL = AppURL - setting.AppSubURL = AppSubURL - - test := func(input, expected string) { - result := RenderFullIssuePattern([]byte(input)) - assert.Equal(t, expected, string(result)) - } - test("Here is a link https://git.osgeo.org/gogs/postgis/postgis/pulls/6", - "Here is a link https://git.osgeo.org/gogs/postgis/postgis/pulls/6") - test("Look here http://localhost:3000/person/repo/issues/4", - `Look here <a href="http://localhost:3000/person/repo/issues/4">#4</a>`) - test("http://localhost:3000/person/repo/issues/4#issuecomment-1234", - `<a href="http://localhost:3000/person/repo/issues/4#issuecomment-1234">#4</a>`) -} - -func TestRegExp_MentionPattern(t *testing.T) { - trueTestCases := []string{ - "@Unknwon", - "@ANT_123", - "@xxx-DiN0-z-A..uru..s-xxx", - " @lol ", - " @Te/st", - } - falseTestCases := []string{ - "@ 0", - "@ ", - "@", - "", - "ABC", - } - - for _, testCase := range trueTestCases { - res := MentionPattern.MatchString(testCase) - if !res { - println() - println(testCase) - } - assert.True(t, res) - } - for _, testCase := range falseTestCases { - res := MentionPattern.MatchString(testCase) - if res { - println() - println(testCase) - } - assert.False(t, res) - } -} - -func TestRegExp_IssueNumericPattern(t *testing.T) { - trueTestCases := []string{ - "#1234", - "#0", - "#1234567890987654321", - "[#1234]", - } - falseTestCases := []string{ - "# 1234", - "# 0", - "# ", - "#", - "#ABC", - "#1A2B", - "", - "ABC", - "[]", - "[x]", - } - - for _, testCase := range trueTestCases { - assert.True(t, IssueNumericPattern.MatchString(testCase)) - } - for _, testCase := range falseTestCases { - assert.False(t, IssueNumericPattern.MatchString(testCase)) - } -} - -func TestRegExp_IssueAlphanumericPattern(t *testing.T) { - trueTestCases := []string{ - "ABC-1234", - "A-1", - "RC-80", - "ABCDEFGHIJ-1234567890987654321234567890", - "[JIRA-134]", - } - falseTestCases := []string{ - "RC-08", - "PR-0", - "ABCDEFGHIJK-1", - "PR_1", - "", - "#ABC", - "", - "ABC", - "GG-", - "rm-1", - "[]", - } - - for _, testCase := range trueTestCases { - assert.True(t, IssueAlphanumericPattern.MatchString(testCase)) - } - for _, testCase := range falseTestCases { - assert.False(t, IssueAlphanumericPattern.MatchString(testCase)) - } -} - -func TestRegExp_Sha1CurrentPattern(t *testing.T) { - trueTestCases := []string{ - "d8a994ef243349f321568f9e36d5c3f444b99cae", - "abcdefabcdefabcdefabcdefabcdefabcdefabcd", - } - falseTestCases := []string{ - "test", - "abcdefg", - "abcdefghijklmnopqrstuvwxyzabcdefghijklmn", - "abcdefghijklmnopqrstuvwxyzabcdefghijklmO", - } - - for _, testCase := range trueTestCases { - assert.True(t, Sha1CurrentPattern.MatchString(testCase)) - } - for _, testCase := range falseTestCases { - assert.False(t, Sha1CurrentPattern.MatchString(testCase)) - } -} - -func TestRegExp_AnySHA1Pattern(t *testing.T) { - testCases := map[string][]string{ - "https://github.com/jquery/jquery/blob/a644101ed04d0beacea864ce805e0c4f86ba1cd1/test/unit/event.js#L2703": { - "https", - "github.com", - "jquery", - "jquery", - "blob", - "a644101ed04d0beacea864ce805e0c4f86ba1cd1", - "test/unit/event.js", - "L2703", - }, - "https://github.com/jquery/jquery/blob/a644101ed04d0beacea864ce805e0c4f86ba1cd1/test/unit/event.js": { - "https", - "github.com", - "jquery", - "jquery", - "blob", - "a644101ed04d0beacea864ce805e0c4f86ba1cd1", - "test/unit/event.js", - "", - }, - "https://github.com/jquery/jquery/commit/0705be475092aede1eddae01319ec931fb9c65fc": { - "https", - "github.com", - "jquery", - "jquery", - "commit", - "0705be475092aede1eddae01319ec931fb9c65fc", - "", - "", - }, - "https://github.com/jquery/jquery/tree/0705be475092aede1eddae01319ec931fb9c65fc/src": { - "https", - "github.com", - "jquery", - "jquery", - "tree", - "0705be475092aede1eddae01319ec931fb9c65fc", - "src", - "", - }, - "https://try.gogs.io/gogs/gogs/commit/d8a994ef243349f321568f9e36d5c3f444b99cae#diff-2": { - "https", - "try.gogs.io", - "gogs", - "gogs", - "commit", - "d8a994ef243349f321568f9e36d5c3f444b99cae", - "", - "diff-2", - }, - } - - for k, v := range testCases { - assert.Equal(t, AnySHA1Pattern.FindStringSubmatch(k)[1:], v) - } -} - func TestMisc_IsSameDomain(t *testing.T) { setting.AppURL = AppURL setting.AppSubURL = AppSubURL @@ -464,3 +66,66 @@ func TestMisc_IsSameDomain(t *testing.T) { assert.False(t, IsSameDomain("http://google.com/ncr")) assert.False(t, IsSameDomain("favicon.ico")) } + +func TestRender_ShortLinks(t *testing.T) { + setting.AppURL = AppURL + setting.AppSubURL = AppSubURL + tree := util.URLJoin(AppSubURL, "src", "master") + + test := func(input, expected, expectedWiki string) { + buffer := markdown.RenderString(input, tree, nil) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer))) + buffer = markdown.RenderWiki([]byte(input), setting.AppSubURL, nil) + assert.Equal(t, strings.TrimSpace(expectedWiki), strings.TrimSpace(string(buffer))) + } + + rawtree := util.URLJoin(AppSubURL, "raw", "master") + url := util.URLJoin(tree, "Link") + otherURL := util.URLJoin(tree, "OtherLink") + imgurl := util.URLJoin(rawtree, "Link.jpg") + urlWiki := util.URLJoin(AppSubURL, "wiki", "Link") + otherURLWiki := util.URLJoin(AppSubURL, "wiki", "OtherLink") + imgurlWiki := util.URLJoin(AppSubURL, "wiki", "raw", "Link.jpg") + favicon := "http://google.com/favicon.ico" + + test( + "[[Link]]", + `<p><a href="`+url+`" rel="nofollow">Link</a></p>`, + `<p><a href="`+urlWiki+`" rel="nofollow">Link</a></p>`) + test( + "[[Link.jpg]]", + `<p><a href="`+imgurl+`" rel="nofollow"><img src="`+imgurl+`" title="Link.jpg" alt="Link.jpg"/></a></p>`, + `<p><a href="`+imgurlWiki+`" rel="nofollow"><img src="`+imgurlWiki+`" title="Link.jpg" alt="Link.jpg"/></a></p>`) + test( + "[["+favicon+"]]", + `<p><a href="`+favicon+`" rel="nofollow"><img src="`+favicon+`" title="favicon.ico"/></a></p>`, + `<p><a href="`+favicon+`" rel="nofollow"><img src="`+favicon+`" title="favicon.ico"/></a></p>`) + test( + "[[Name|Link]]", + `<p><a href="`+url+`" rel="nofollow">Name</a></p>`, + `<p><a href="`+urlWiki+`" rel="nofollow">Name</a></p>`) + test( + "[[Name|Link.jpg]]", + `<p><a href="`+imgurl+`" rel="nofollow"><img src="`+imgurl+`" title="Name" alt="Name"/></a></p>`, + `<p><a href="`+imgurlWiki+`" rel="nofollow"><img src="`+imgurlWiki+`" title="Name" alt="Name"/></a></p>`) + test( + "[[Name|Link.jpg|alt=AltName]]", + `<p><a href="`+imgurl+`" rel="nofollow"><img src="`+imgurl+`" title="AltName" alt="AltName"/></a></p>`, + `<p><a href="`+imgurlWiki+`" rel="nofollow"><img src="`+imgurlWiki+`" title="AltName" alt="AltName"/></a></p>`) + test( + "[[Name|Link.jpg|title=Title]]", + `<p><a href="`+imgurl+`" rel="nofollow"><img src="`+imgurl+`" title="Title" alt="Title"/></a></p>`, + `<p><a href="`+imgurlWiki+`" rel="nofollow"><img src="`+imgurlWiki+`" title="Title" alt="Title"/></a></p>`) + test( + "[[Name|Link.jpg|alt=AltName|title=Title]]", + `<p><a href="`+imgurl+`" rel="nofollow"><img src="`+imgurl+`" title="Title" alt="AltName"/></a></p>`, + `<p><a href="`+imgurlWiki+`" rel="nofollow"><img src="`+imgurlWiki+`" title="Title" alt="AltName"/></a></p>`) + test( + "[[Name|Link.jpg|alt=\"AltName\"|title='Title']]", + `<p><a href="`+imgurl+`" rel="nofollow"><img src="`+imgurl+`" title="Title" alt="AltName"/></a></p>`, + `<p><a href="`+imgurlWiki+`" rel="nofollow"><img src="`+imgurlWiki+`" title="Title" alt="AltName"/></a></p>`) + test( + "[[Link]] [[OtherLink]]", + `<p><a href="`+url+`" rel="nofollow">Link</a> <a href="`+otherURL+`" rel="nofollow">OtherLink</a></p>`, + `<p><a href="`+urlWiki+`" rel="nofollow">Link</a> <a href="`+otherURLWiki+`" rel="nofollow">OtherLink</a></p>`) +} diff --git a/modules/markup/markdown/markdown.go b/modules/markup/markdown/markdown.go index 2e3d180c4e..901edb80a7 100644 --- a/modules/markup/markdown/markdown.go +++ b/modules/markup/markdown/markdown.go @@ -22,17 +22,20 @@ type Renderer struct { IsWiki bool } +var byteMailto = []byte("mailto:") + // Link defines how formal links should be processed to produce corresponding HTML elements. func (r *Renderer) Link(out *bytes.Buffer, link []byte, title []byte, content []byte) { - if len(link) > 0 && !markup.IsLink(link) { - if link[0] != '#' { - lnk := string(link) - if r.IsWiki { - lnk = util.URLJoin("wiki", lnk) - } - mLink := util.URLJoin(r.URLPrefix, lnk) - link = []byte(mLink) + // special case: this is not a link, a hash link or a mailto:, so it's a + // relative URL + if len(link) > 0 && !markup.IsLink(link) && + link[0] != '#' && !bytes.HasPrefix(link, byteMailto) { + lnk := string(link) + if r.IsWiki { + lnk = util.URLJoin("wiki", lnk) } + mLink := util.URLJoin(r.URLPrefix, lnk) + link = []byte(mLink) } r.Renderer.Link(out, link, title, content) @@ -124,30 +127,33 @@ func (r *Renderer) Image(out *bytes.Buffer, link []byte, title []byte, alt []byt out.WriteString("</a>") } +const ( + blackfridayExtensions = 0 | + blackfriday.EXTENSION_NO_INTRA_EMPHASIS | + blackfriday.EXTENSION_TABLES | + blackfriday.EXTENSION_FENCED_CODE | + blackfriday.EXTENSION_STRIKETHROUGH | + blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK + blackfridayHTMLFlags = 0 | + blackfriday.HTML_SKIP_STYLE | + blackfriday.HTML_OMIT_CONTENTS | + blackfriday.HTML_USE_SMARTYPANTS +) + // RenderRaw renders Markdown to HTML without handling special links. func RenderRaw(body []byte, urlPrefix string, wikiMarkdown bool) []byte { - htmlFlags := 0 - htmlFlags |= blackfriday.HTML_SKIP_STYLE - htmlFlags |= blackfriday.HTML_OMIT_CONTENTS renderer := &Renderer{ - Renderer: blackfriday.HtmlRenderer(htmlFlags, "", ""), + Renderer: blackfriday.HtmlRenderer(blackfridayHTMLFlags, "", ""), URLPrefix: urlPrefix, IsWiki: wikiMarkdown, } - // set up the parser - extensions := 0 - extensions |= blackfriday.EXTENSION_NO_INTRA_EMPHASIS - extensions |= blackfriday.EXTENSION_TABLES - extensions |= blackfriday.EXTENSION_FENCED_CODE - extensions |= blackfriday.EXTENSION_STRIKETHROUGH - extensions |= blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK - + exts := blackfridayExtensions if setting.Markdown.EnableHardLineBreak { - extensions |= blackfriday.EXTENSION_HARD_LINE_BREAK + exts |= blackfriday.EXTENSION_HARD_LINE_BREAK } - body = blackfriday.Markdown(body, renderer, extensions) + body = blackfriday.Markdown(body, renderer, exts) return body } diff --git a/modules/markup/markdown/markdown_test.go b/modules/markup/markdown/markdown_test.go index c19037f629..605094df46 100644 --- a/modules/markup/markdown/markdown_test.go +++ b/modules/markup/markdown/markdown_test.go @@ -8,7 +8,6 @@ import ( "strings" "testing" - "code.gitea.io/gitea/modules/markup" . "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -41,69 +40,6 @@ func TestRender_StandardLinks(t *testing.T) { `<p><a href="`+lnkWiki+`" rel="nofollow">WikiPage</a></p>`) } -func TestRender_ShortLinks(t *testing.T) { - setting.AppURL = AppURL - setting.AppSubURL = AppSubURL - tree := util.URLJoin(AppSubURL, "src", "master") - - test := func(input, expected, expectedWiki string) { - buffer := RenderString(input, tree, nil) - assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer))) - buffer = RenderWiki([]byte(input), setting.AppSubURL, nil) - assert.Equal(t, strings.TrimSpace(expectedWiki), strings.TrimSpace(string(buffer))) - } - - rawtree := util.URLJoin(AppSubURL, "raw", "master") - url := util.URLJoin(tree, "Link") - otherUrl := util.URLJoin(tree, "OtherLink") - imgurl := util.URLJoin(rawtree, "Link.jpg") - urlWiki := util.URLJoin(AppSubURL, "wiki", "Link") - otherUrlWiki := util.URLJoin(AppSubURL, "wiki", "OtherLink") - imgurlWiki := util.URLJoin(AppSubURL, "wiki", "raw", "Link.jpg") - favicon := "http://google.com/favicon.ico" - - test( - "[[Link]]", - `<p><a href="`+url+`" rel="nofollow">Link</a></p>`, - `<p><a href="`+urlWiki+`" rel="nofollow">Link</a></p>`) - test( - "[[Link.jpg]]", - `<p><a href="`+imgurl+`" rel="nofollow"><img src="`+imgurl+`" alt="Link.jpg" title="Link.jpg"/></a></p>`, - `<p><a href="`+imgurlWiki+`" rel="nofollow"><img src="`+imgurlWiki+`" alt="Link.jpg" title="Link.jpg"/></a></p>`) - test( - "[["+favicon+"]]", - `<p><a href="`+favicon+`" rel="nofollow"><img src="`+favicon+`" title="favicon.ico"/></a></p>`, - `<p><a href="`+favicon+`" rel="nofollow"><img src="`+favicon+`" title="favicon.ico"/></a></p>`) - test( - "[[Name|Link]]", - `<p><a href="`+url+`" rel="nofollow">Name</a></p>`, - `<p><a href="`+urlWiki+`" rel="nofollow">Name</a></p>`) - test( - "[[Name|Link.jpg]]", - `<p><a href="`+imgurl+`" rel="nofollow"><img src="`+imgurl+`" alt="Name" title="Name"/></a></p>`, - `<p><a href="`+imgurlWiki+`" rel="nofollow"><img src="`+imgurlWiki+`" alt="Name" title="Name"/></a></p>`) - test( - "[[Name|Link.jpg|alt=AltName]]", - `<p><a href="`+imgurl+`" rel="nofollow"><img src="`+imgurl+`" alt="AltName" title="AltName"/></a></p>`, - `<p><a href="`+imgurlWiki+`" rel="nofollow"><img src="`+imgurlWiki+`" alt="AltName" title="AltName"/></a></p>`) - test( - "[[Name|Link.jpg|title=Title]]", - `<p><a href="`+imgurl+`" rel="nofollow"><img src="`+imgurl+`" alt="Title" title="Title"/></a></p>`, - `<p><a href="`+imgurlWiki+`" rel="nofollow"><img src="`+imgurlWiki+`" alt="Title" title="Title"/></a></p>`) - test( - "[[Name|Link.jpg|alt=AltName|title=Title]]", - `<p><a href="`+imgurl+`" rel="nofollow"><img src="`+imgurl+`" alt="AltName" title="Title"/></a></p>`, - `<p><a href="`+imgurlWiki+`" rel="nofollow"><img src="`+imgurlWiki+`" alt="AltName" title="Title"/></a></p>`) - test( - "[[Name|Link.jpg|alt=\"AltName\"|title='Title']]", - `<p><a href="`+imgurl+`" rel="nofollow"><img src="`+imgurl+`" alt="AltName" title="Title"/></a></p>`, - `<p><a href="`+imgurlWiki+`" rel="nofollow"><img src="`+imgurlWiki+`" alt="AltName" title="Title"/></a></p>`) - test( - "[[Link]] [[OtherLink]]", - `<p><a href="`+url+`" rel="nofollow">Link</a> <a href="`+otherUrl+`" rel="nofollow">OtherLink</a></p>`, - `<p><a href="`+urlWiki+`" rel="nofollow">Link</a> <a href="`+otherUrlWiki+`" rel="nofollow">OtherLink</a></p>`) -} - func TestMisc_IsMarkdownFile(t *testing.T) { setting.Markdown.FileExtensions = []string{".md", ".markdown", ".mdown", ".mkd"} trueTestCases := []string{ @@ -141,35 +77,11 @@ func TestRender_Images(t *testing.T) { test( "!["+title+"]("+url+")", - `<p><a href="`+result+`" rel="nofollow"><img src="`+result+`" alt="`+title+`"></a></p>`) + `<p><a href="`+result+`" rel="nofollow"><img src="`+result+`" alt="`+title+`"/></a></p>`) test( "[["+title+"|"+url+"]]", - `<p><a href="`+result+`" rel="nofollow"><img src="`+result+`" alt="`+title+`" title="`+title+`"/></a></p>`) -} - -func TestRegExp_ShortLinkPattern(t *testing.T) { - trueTestCases := []string{ - "[[stuff]]", - "[[]]", - "[[stuff|title=Difficult name with spaces*!]]", - } - falseTestCases := []string{ - "test", - "abcdefg", - "[[]", - "[[", - "[]", - "]]", - "abcdefghijklmnopqrstuvwxyz", - } - - for _, testCase := range trueTestCases { - assert.True(t, markup.ShortLinkPattern.MatchString(testCase)) - } - for _, testCase := range falseTestCases { - assert.False(t, markup.ShortLinkPattern.MatchString(testCase)) - } + `<p><a href="`+result+`" rel="nofollow"><img src="`+result+`" title="`+title+`" alt="`+title+`"/></a></p>`) } func testAnswers(baseURLContent, baseURLImages string) []string { @@ -185,7 +97,7 @@ func testAnswers(baseURLContent, baseURLImages string) []string { <ul> <li>Bezier widget (by <a href="` + AppURL + `r-lyeh" rel="nofollow">@r-lyeh</a>) <a href="http://localhost:3000/ocornut/imgui/issues/786" rel="nofollow">#786</a></li> -<li>Node graph editors https://github.com/ocornut/imgui/issues/306</li> +<li>Node graph editors <a href="https://github.com/ocornut/imgui/issues/306" rel="nofollow">https://github.com/ocornut/imgui/issues/306</a></li> <li><a href="` + baseURLContent + `/memory_editor_example" rel="nofollow">Memory Editor</a></li> <li><a href="` + baseURLContent + `/plot_var_example" rel="nofollow">Plot var helper</a></li> </ul> @@ -201,14 +113,14 @@ func testAnswers(baseURLContent, baseURLImages string) []string { <table> <thead> <tr> -<th><a href="` + baseURLImages + `/images/icon-install.png" rel="nofollow"><img src="` + baseURLImages + `/images/icon-install.png" alt="images/icon-install.png" title="icon-install.png"/></a></th> +<th><a href="` + baseURLImages + `/images/icon-install.png" rel="nofollow"><img src="` + baseURLImages + `/images/icon-install.png" title="icon-install.png" alt="images/icon-install.png"/></a></th> <th><a href="` + baseURLContent + `/Installation" rel="nofollow">Installation</a></th> </tr> </thead> <tbody> <tr> -<td><a href="` + baseURLImages + `/images/icon-usage.png" rel="nofollow"><img src="` + baseURLImages + `/images/icon-usage.png" alt="images/icon-usage.png" title="icon-usage.png"/></a></td> +<td><a href="` + baseURLImages + `/images/icon-usage.png" rel="nofollow"><img src="` + baseURLImages + `/images/icon-usage.png" title="icon-usage.png" alt="images/icon-usage.png"/></a></td> <td><a href="` + baseURLContent + `/Usage" rel="nofollow">Usage</a></td> </tr> </tbody> @@ -218,9 +130,9 @@ func testAnswers(baseURLContent, baseURLImages string) []string { <ol> <li><a href="https://github.com/libgdx/libgdx/wiki/Gradle-on-the-Commandline#packaging-for-the-desktop" rel="nofollow">Package your libGDX application</a> -<a href="` + baseURLImages + `/images/1.png" rel="nofollow"><img src="` + baseURLImages + `/images/1.png" alt="images/1.png" title="1.png"/></a></li> +<a href="` + baseURLImages + `/images/1.png" rel="nofollow"><img src="` + baseURLImages + `/images/1.png" title="1.png" alt="images/1.png"/></a></li> <li>Perform a test run by hitting the Run! button. -<a href="` + baseURLImages + `/images/2.png" rel="nofollow"><img src="` + baseURLImages + `/images/2.png" alt="images/2.png" title="2.png"/></a></li> +<a href="` + baseURLImages + `/images/2.png" rel="nofollow"><img src="` + baseURLImages + `/images/2.png" title="2.png" alt="images/2.png"/></a></li> </ol> `, } diff --git a/modules/markup/markup.go b/modules/markup/markup.go index ba28ec53c0..d17270fb01 100644 --- a/modules/markup/markup.go +++ b/modules/markup/markup.go @@ -7,6 +7,8 @@ package markup import ( "path/filepath" "strings" + + "code.gitea.io/gitea/modules/log" ) // Init initialize regexps for markdown parsing @@ -69,7 +71,11 @@ func RenderWiki(filename string, rawBytes []byte, urlPrefix string, metas map[st func render(parser Parser, rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte { urlPrefix = strings.Replace(urlPrefix, " ", "+", -1) result := parser.Render(rawBytes, urlPrefix, metas, isWiki) - result = PostProcess(result, urlPrefix, metas, isWiki) + // TODO: one day the error should be returned. + result, err := PostProcess(result, urlPrefix, metas, isWiki) + if err != nil { + log.Error(3, "PostProcess: %v", err) + } return SanitizeBytes(result) } diff --git a/modules/templates/helper.go b/modules/templates/helper.go index 3f3d6083f2..98900c7538 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -10,6 +10,7 @@ import ( "encoding/json" "errors" "fmt" + "html" "html/template" "mime" "net/url" @@ -27,7 +28,6 @@ import ( "golang.org/x/net/html/charset" "golang.org/x/text/transform" "gopkg.in/editorconfig/editorconfig-core-go.v1" - "html" ) // NewFuncMap returns functions for injecting to templates @@ -280,26 +280,21 @@ func ReplaceLeft(s, old, new string) string { // RenderCommitMessage renders commit message with XSS-safe and special links. func RenderCommitMessage(msg, urlPrefix string, metas map[string]string) template.HTML { - return renderCommitMessage(msg, markup.RenderIssueIndexPatternOptions{ - URLPrefix: urlPrefix, - Metas: metas, - }) + return RenderCommitMessageLink(msg, urlPrefix, "", metas) } // RenderCommitMessageLink renders commit message as a XXS-safe link to the provided // default url, handling for special links. -func RenderCommitMessageLink(msg, urlPrefix string, urlDefault string, metas map[string]string) template.HTML { - return renderCommitMessage(msg, markup.RenderIssueIndexPatternOptions{ - DefaultURL: urlDefault, - URLPrefix: urlPrefix, - Metas: metas, - }) -} - -func renderCommitMessage(msg string, opts markup.RenderIssueIndexPatternOptions) template.HTML { +func RenderCommitMessageLink(msg, urlPrefix, urlDefault string, metas map[string]string) template.HTML { cleanMsg := template.HTMLEscapeString(msg) - fullMessage := string(markup.RenderIssueIndexPattern([]byte(cleanMsg), opts)) - msgLines := strings.Split(strings.TrimSpace(fullMessage), "\n") + // we can safely assume that it will not return any error, since there + // shouldn't be any special HTML. + fullMessage, err := markup.RenderCommitMessage([]byte(cleanMsg), urlPrefix, urlDefault, metas) + if err != nil { + log.Error(3, "RenderCommitMessage: %v", err) + return "" + } + msgLines := strings.Split(strings.TrimSpace(string(fullMessage)), "\n") if len(msgLines) == 0 { return template.HTML("") } @@ -308,16 +303,13 @@ func renderCommitMessage(msg string, opts markup.RenderIssueIndexPatternOptions) // RenderCommitBody extracts the body of a commit message without its title. func RenderCommitBody(msg, urlPrefix string, metas map[string]string) template.HTML { - return renderCommitBody(msg, markup.RenderIssueIndexPatternOptions{ - URLPrefix: urlPrefix, - Metas: metas, - }) -} - -func renderCommitBody(msg string, opts markup.RenderIssueIndexPatternOptions) template.HTML { cleanMsg := template.HTMLEscapeString(msg) - fullMessage := string(markup.RenderIssueIndexPattern([]byte(cleanMsg), opts)) - body := strings.Split(strings.TrimSpace(fullMessage), "\n") + fullMessage, err := markup.RenderCommitMessage([]byte(cleanMsg), urlPrefix, "", metas) + if err != nil { + log.Error(3, "RenderCommitMessage: %v", err) + return "" + } + body := strings.Split(strings.TrimSpace(string(fullMessage)), "\n") if len(body) == 0 { return template.HTML("") } diff --git a/public/js/index.js b/public/js/index.js index 137e50f9ce..5d3d2a013d 100644 --- a/public/js/index.js +++ b/public/js/index.js @@ -771,7 +771,6 @@ function initWikiForm() { function (data) { preview.innerHTML = '<div class="markdown">' + data + '</div>'; emojify.run($('.editor-preview')[0]); - $('.editor-preview').autolink(); } ); }, 0); @@ -1549,7 +1548,6 @@ $(document).ready(function () { node.append('<a class="anchor" href="#' + name + '"><span class="octicon octicon-link"></span></a>'); }); }); - $('.markdown').autolink(); $('.issue-checkbox').click(function() { var numChecked = $('.issue-checkbox').children('input:checked').length; diff --git a/public/vendor/plugins/autolink/LICENSE b/public/vendor/plugins/autolink/LICENSE deleted file mode 100644 index bb35e8b86d..0000000000 --- a/public/vendor/plugins/autolink/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2015 egoist 0x142857@gmail.com - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE.
\ No newline at end of file diff --git a/public/vendor/plugins/autolink/autolink.js b/public/vendor/plugins/autolink/autolink.js deleted file mode 100644 index 2993954ab6..0000000000 --- a/public/vendor/plugins/autolink/autolink.js +++ /dev/null @@ -1,45 +0,0 @@ -(function () { - var re = /((([A-Za-z]{3,9}:(?:\/\/)?)(?:[\-;:&=\+\$,\w]+@)?[A-Za-z0-9\.\-]+|(?:www\.|[\-;:&=\+\$,\w]+@)[A-Za-z0-9\.\-]+)((?:\/[\+~%\/\.\w\-]*)?\??(?:[\-\+:=&;%@\.\w]*)#?(?:[\.\!\/\\\w]*))?)/g; - function textNodesUnder(node) { - var textNodes = []; - if(typeof document.createTreeWalker === 'function') { - // Efficient TreeWalker - var currentNode, walker; - walker = document.createTreeWalker(node, NodeFilter.SHOW_TEXT, null, false); - while(currentNode = walker.nextNode()) { - textNodes.push(currentNode); - } - } else { - // Less efficient recursive function - for(node = node.firstChild; node; node = node.nextSibling) { - if(node.nodeType === 3) { - textNodes.push(node); - } else { - textNodes = textNodes.concat(textNodesUnder(node)); - } - } - } - return textNodes; - } - - function processNode(node) { - re.lastIndex = 0; - var results = re.exec(node.textContent); - if(results !== null) { - if($(node).parents().filter('code').length === 0) { - $(node).replaceWith( - $('<span />').html( - node.nodeValue.replace(re, '<a href="$1">$1</a>') - ) - ); - } - } - } - - jQuery.fn.autolink = function () { - this.each(function () { - textNodesUnder(this).forEach(processNode); - }); - return this; - }; -})(); diff --git a/routers/api/v1/misc/markdown_test.go b/routers/api/v1/misc/markdown_test.go index c1449589a3..f9503bc639 100644 --- a/routers/api/v1/misc/markdown_test.go +++ b/routers/api/v1/misc/markdown_test.go @@ -73,7 +73,7 @@ func TestAPI_RenderGFM(t *testing.T) { <ul> <li><a href="` + AppSubURL + `wiki/Links" rel="nofollow">Links, Language bindings, Engine bindings</a></li> <li><a href="` + AppSubURL + `wiki/Tips" rel="nofollow">Tips</a></li> -<li>Bezier widget (by <a href="` + AppURL + `r-lyeh" rel="nofollow">@r-lyeh</a>) https://github.com/ocornut/imgui/issues/786</li> +<li>Bezier widget (by <a href="` + AppURL + `r-lyeh" rel="nofollow">@r-lyeh</a>) <a href="https://github.com/ocornut/imgui/issues/786" rel="nofollow">https://github.com/ocornut/imgui/issues/786</a></li> </ul> `, // wine-staging wiki home extract: special wiki syntax, images @@ -96,7 +96,7 @@ Here are some links to the most important topics. You can find the full list of <p>Here are some links to the most important topics. You can find the full list of pages at the sidebar.</p> <p><a href="` + AppSubURL + `wiki/Configuration" rel="nofollow">Configuration</a> -<a href="` + AppSubURL + `wiki/raw/images/icon-bug.png" rel="nofollow"><img src="` + AppSubURL + `wiki/raw/images/icon-bug.png" alt="images/icon-bug.png" title="icon-bug.png"/></a></p> +<a href="` + AppSubURL + `wiki/raw/images/icon-bug.png" rel="nofollow"><img src="` + AppSubURL + `wiki/raw/images/icon-bug.png" title="icon-bug.png" alt="images/icon-bug.png"/></a></p> `, // Guard wiki sidebar: special syntax `[[Guardfile-DSL / Configuring-Guard|Guardfile-DSL---Configuring-Guard]]`, diff --git a/templates/base/footer.tmpl b/templates/base/footer.tmpl index 18d59d92c5..c0f5a83d7b 100644 --- a/templates/base/footer.tmpl +++ b/templates/base/footer.tmpl @@ -122,7 +122,6 @@ emojiTribute.attach(document.getElementById('content')) </script> {{end}} - <script src="{{AppSubUrl}}/vendor/plugins/autolink/autolink.js"></script> <script src="{{AppSubUrl}}/vendor/plugins/emojify/emojify.min.js"></script> <script src="{{AppSubUrl}}/vendor/plugins/clipboard/clipboard.min.js"></script> <script src="{{AppSubUrl}}/vendor/plugins/vue/vue.min.js"></script> |