diff options
Diffstat (limited to 'modules/markup')
50 files changed, 705 insertions, 555 deletions
diff --git a/modules/markup/common/footnote.go b/modules/markup/common/footnote.go index 4406803694..1ece436c66 100644 --- a/modules/markup/common/footnote.go +++ b/modules/markup/common/footnote.go @@ -53,7 +53,7 @@ type FootnoteLink struct { // Dump implements Node.Dump. func (n *FootnoteLink) Dump(source []byte, level int) { m := map[string]string{} - m["Index"] = fmt.Sprintf("%v", n.Index) + m["Index"] = strconv.Itoa(n.Index) m["Name"] = fmt.Sprintf("%v", n.Name) ast.DumpHelper(n, source, level, m, nil) } @@ -85,7 +85,7 @@ type FootnoteBackLink struct { // Dump implements Node.Dump. func (n *FootnoteBackLink) Dump(source []byte, level int) { m := map[string]string{} - m["Index"] = fmt.Sprintf("%v", n.Index) + m["Index"] = strconv.Itoa(n.Index) m["Name"] = fmt.Sprintf("%v", n.Name) ast.DumpHelper(n, source, level, m, nil) } @@ -151,7 +151,7 @@ type FootnoteList struct { // Dump implements Node.Dump. func (n *FootnoteList) Dump(source []byte, level int) { m := map[string]string{} - m["Count"] = fmt.Sprintf("%v", n.Count) + m["Count"] = strconv.Itoa(n.Count) ast.DumpHelper(n, source, level, m, nil) } @@ -197,7 +197,7 @@ func (b *footnoteBlockParser) Open(parent ast.Node, reader text.Reader, pc parse return nil, parser.NoChildren } open := pos + 1 - closure := util.FindClosure(line[pos+1:], '[', ']', false, false) //nolint + closure := util.FindClosure(line[pos+1:], '[', ']', false, false) //nolint:staticcheck // deprecated function closes := pos + 1 + closure next := closes + 1 if closure > -1 { @@ -287,7 +287,7 @@ func (s *footnoteParser) Parse(parent ast.Node, block text.Reader, pc parser.Con return nil } open := pos - closure := util.FindClosure(line[pos:], '[', ']', false, false) //nolint + closure := util.FindClosure(line[pos:], '[', ']', false, false) //nolint:staticcheck // deprecated function if closure < 0 { return nil } @@ -409,9 +409,9 @@ func (r *FootnoteHTMLRenderer) renderFootnoteLink(w util.BufWriter, source []byt _, _ = w.Write(n.Name) _, _ = w.WriteString(`"><a href="#fn:`) _, _ = w.Write(n.Name) - _, _ = w.WriteString(`" class="footnote-ref" role="doc-noteref">`) + _, _ = w.WriteString(`" class="footnote-ref" role="doc-noteref">`) // FIXME: here and below, need to keep the classes _, _ = w.WriteString(is) - _, _ = w.WriteString(`</a></sup>`) + _, _ = w.WriteString(` </a></sup>`) // the style doesn't work at the moment, so add a space to separate the names } return ast.WalkContinue, nil } diff --git a/modules/markup/common/linkify.go b/modules/markup/common/linkify.go index 52888958fa..3eecb97eac 100644 --- a/modules/markup/common/linkify.go +++ b/modules/markup/common/linkify.go @@ -85,9 +85,10 @@ func (s *linkifyParser) Parse(parent ast.Node, block text.Reader, pc parser.Cont } else if lastChar == ')' { closing := 0 for i := m[1] - 1; i >= m[0]; i-- { - if line[i] == ')' { + switch line[i] { + case ')': closing++ - } else if line[i] == '(' { + case '(': closing-- } } diff --git a/modules/markup/console/console.go b/modules/markup/console/console.go index 06f3acfa68..492579b0a5 100644 --- a/modules/markup/console/console.go +++ b/modules/markup/console/console.go @@ -6,13 +6,14 @@ package console import ( "bytes" "io" - "path" + "unicode/utf8" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/typesniffer" + "code.gitea.io/gitea/modules/util" trend "github.com/buildkite/terminal-to-html/v3" - "github.com/go-enry/go-enry/v2" ) func init() { @@ -22,6 +23,8 @@ func init() { // Renderer implements markup.Renderer type Renderer struct{} +var _ markup.RendererContentDetector = (*Renderer)(nil) + // Name implements markup.Renderer func (Renderer) Name() string { return "console" @@ -40,15 +43,36 @@ func (Renderer) SanitizerRules() []setting.MarkupSanitizerRule { } // CanRender implements markup.RendererContentDetector -func (Renderer) CanRender(filename string, input io.Reader) bool { - buf, err := io.ReadAll(input) - if err != nil { +func (Renderer) CanRender(filename string, sniffedType typesniffer.SniffedType, prefetchBuf []byte) bool { + if !sniffedType.IsTextPlain() { return false } - if enry.GetLanguage(path.Base(filename), buf) != enry.OtherLanguage { + + s := util.UnsafeBytesToString(prefetchBuf) + rs := []rune(s) + cnt := 0 + firstErrPos := -1 + isCtrlSep := func(p int) bool { + return p < len(rs) && (rs[p] == ';' || rs[p] == 'm') + } + for i, c := range rs { + if c == 0 { + return false + } + if c == '\x1b' { + match := i+1 < len(rs) && rs[i+1] == '[' + if match && (isCtrlSep(i+2) || isCtrlSep(i+3) || isCtrlSep(i+4) || isCtrlSep(i+5)) { + cnt++ + } + } + if c == utf8.RuneError && firstErrPos == -1 { + firstErrPos = i + } + } + if firstErrPos != -1 && firstErrPos != len(rs)-1 { return false } - return bytes.ContainsRune(buf, '\x1b') + return cnt >= 2 // only render it as console output if there are at least two escape sequences } // Render renders terminal colors to HTML with all specific handling stuff. diff --git a/modules/markup/console/console_test.go b/modules/markup/console/console_test.go index e1f0da1f01..d1192bebc2 100644 --- a/modules/markup/console/console_test.go +++ b/modules/markup/console/console_test.go @@ -4,28 +4,43 @@ package console import ( - "context" "strings" "testing" "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/typesniffer" "github.com/stretchr/testify/assert" ) func TestRenderConsole(t *testing.T) { - var render Renderer - kases := map[string]string{ - "\x1b[37m\x1b[40mnpm\x1b[0m \x1b[0m\x1b[32minfo\x1b[0m \x1b[0m\x1b[35mit worked if it ends with\x1b[0m ok": "<span class=\"term-fg37 term-bg40\">npm</span> <span class=\"term-fg32\">info</span> <span class=\"term-fg35\">it worked if it ends with</span> ok", + cases := []struct { + input string + expected string + }{ + {"\x1b[37m\x1b[40mnpm\x1b[0m \x1b[0m\x1b[32minfo\x1b[0m \x1b[0m\x1b[35mit worked if it ends with\x1b[0m ok", `<span class="term-fg37 term-bg40">npm</span> <span class="term-fg32">info</span> <span class="term-fg35">it worked if it ends with</span> ok`}, + {"\x1b[1;2m \x1b[123m 啊", `<span class="term-fg2"> 啊</span>`}, + {"\x1b[1;2m \x1b[123m \xef", `<span class="term-fg2"> �</span>`}, + {"\x1b[1;2m \x1b[123m \xef \xef", ``}, + {"\x1b[12", ``}, + {"\x1b[1", ``}, + {"\x1b[FOO\x1b[", ``}, + {"\x1b[mFOO\x1b[m", `FOO`}, } - for k, v := range kases { + var render Renderer + for i, c := range cases { var buf strings.Builder - canRender := render.CanRender("test", strings.NewReader(k)) - assert.True(t, canRender) + st := typesniffer.DetectContentType([]byte(c.input)) + canRender := render.CanRender("test", st, []byte(c.input)) + if c.expected == "" { + assert.False(t, canRender, "case %d: expected not to render", i) + continue + } - err := render.Render(markup.NewRenderContext(context.Background()), strings.NewReader(k), &buf) + assert.True(t, canRender) + err := render.Render(markup.NewRenderContext(t.Context()), strings.NewReader(c.input), &buf) assert.NoError(t, err) - assert.EqualValues(t, v, buf.String()) + assert.Equal(t, c.expected, buf.String()) } } diff --git a/modules/markup/csv/csv_test.go b/modules/markup/csv/csv_test.go index 4c47170c30..fff7f0baca 100644 --- a/modules/markup/csv/csv_test.go +++ b/modules/markup/csv/csv_test.go @@ -4,7 +4,6 @@ package markup import ( - "context" "strings" "testing" @@ -24,8 +23,8 @@ func TestRenderCSV(t *testing.T) { for k, v := range kases { var buf strings.Builder - err := render.Render(markup.NewRenderContext(context.Background()), strings.NewReader(k), &buf) + err := render.Render(markup.NewRenderContext(t.Context()), strings.NewReader(k), &buf) assert.NoError(t, err) - assert.EqualValues(t, v, buf.String()) + assert.Equal(t, v, buf.String()) } } diff --git a/modules/markup/external/external.go b/modules/markup/external/external.go index 03242e569e..39861ade12 100644 --- a/modules/markup/external/external.go +++ b/modules/markup/external/external.go @@ -12,11 +12,9 @@ import ( "runtime" "strings" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" ) // RegisterRenderers registers all supported third part renderers according settings @@ -77,27 +75,22 @@ func envMark(envName string) string { // Render renders the data of the document to HTML via the external tool. func (p *Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { - var ( - command = strings.NewReplacer( - envMark("GITEA_PREFIX_SRC"), ctx.RenderHelper.ResolveLink("", markup.LinkTypeDefault), - envMark("GITEA_PREFIX_RAW"), ctx.RenderHelper.ResolveLink("", markup.LinkTypeRaw), - ).Replace(p.Command) - commands = strings.Fields(command) - args = commands[1:] - ) + baseLinkSrc := ctx.RenderHelper.ResolveLink("", markup.LinkTypeDefault) + baseLinkRaw := ctx.RenderHelper.ResolveLink("", markup.LinkTypeRaw) + command := strings.NewReplacer( + envMark("GITEA_PREFIX_SRC"), baseLinkSrc, + envMark("GITEA_PREFIX_RAW"), baseLinkRaw, + ).Replace(p.Command) + commands := strings.Fields(command) + args := commands[1:] if p.IsInputFile { // write to temp file - f, err := os.CreateTemp("", "gitea_input") + f, cleanup, err := setting.AppDataTempDir("git-repo-content").CreateTempFileRandom("gitea_input") if err != nil { return fmt.Errorf("%s create temp file when rendering %s failed: %w", p.Name(), p.Command, err) } - tmpPath := f.Name() - defer func() { - if err := util.Remove(tmpPath); err != nil { - log.Warn("Unable to remove temporary file: %s: Error: %v", tmpPath, err) - } - }() + defer cleanup() _, err = io.Copy(f, input) if err != nil { @@ -112,14 +105,14 @@ func (p *Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io. args = append(args, f.Name()) } - processCtx, _, finished := process.GetManager().AddContext(ctx, fmt.Sprintf("Render [%s] for %s", commands[0], ctx.RenderHelper.ResolveLink("", markup.LinkTypeDefault))) + processCtx, _, finished := process.GetManager().AddContext(ctx, fmt.Sprintf("Render [%s] for %s", commands[0], baseLinkSrc)) defer finished() cmd := exec.CommandContext(processCtx, commands[0], args...) cmd.Env = append( os.Environ(), - "GITEA_PREFIX_SRC="+ctx.RenderHelper.ResolveLink("", markup.LinkTypeDefault), - "GITEA_PREFIX_RAW="+ctx.RenderHelper.ResolveLink("", markup.LinkTypeRaw), + "GITEA_PREFIX_SRC="+baseLinkSrc, + "GITEA_PREFIX_RAW="+baseLinkRaw, ) if !p.IsInputFile { cmd.Stdin = input diff --git a/modules/markup/html.go b/modules/markup/html.go index bb12febf27..51afd4be00 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "regexp" + "slices" "strings" "sync" @@ -32,7 +33,6 @@ type globalVarsType struct { comparePattern *regexp.Regexp fullURLPattern *regexp.Regexp emailRegex *regexp.Regexp - blackfridayExtRegex *regexp.Regexp emojiShortCodeRegex *regexp.Regexp issueFullPattern *regexp.Regexp filesChangedFullPattern *regexp.Regexp @@ -47,7 +47,7 @@ var globalVars = sync.OnceValue(func() *globalVarsType { // NOTE: All below regex matching do not perform any extra validation. // Thus a link is produced even if the linked entity does not exist. // While fast, this is also incorrect and lead to false positives. - // TODO: fix invalid linking issue + // TODO: fix invalid linking issue (update: stale TODO, what issues? maybe no TODO anymore) // valid chars in encoded path and parameter: [-+~_%.a-zA-Z0-9/] @@ -72,10 +72,8 @@ var globalVars = sync.OnceValue(func() *globalVarsType { // it is still accepted by the CommonMark specification, as well as the HTML5 spec: // http://spec.commonmark.org/0.28/#email-address // https://html.spec.whatwg.org/multipage/input.html#e-mail-state-(type%3Demail) - v.emailRegex = regexp.MustCompile("(?:\\s|^|\\(|\\[)([a-zA-Z0-9.!#$%&'*+\\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9]{2,}(?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+)(?:\\s|$|\\)|\\]|;|,|\\?|!|\\.(\\s|$))") - - // blackfridayExtRegex is for blackfriday extensions create IDs like fn:user-content-footnote - v.blackfridayExtRegex = regexp.MustCompile(`[^:]*:user-content-`) + // At the moment, we use stricter rule for rendering purpose: only allow the "name" part starting after the word boundary + v.emailRegex = regexp.MustCompile(`\b([-\w.!#$%&'*+/=?^{|}~]*@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9]{2,}(?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+)\b`) // emojiShortCodeRegex find emoji by alias like :smile: v.emojiShortCodeRegex = regexp.MustCompile(`:[-+\w]+:`) @@ -89,22 +87,18 @@ var globalVars = sync.OnceValue(func() *globalVarsType { // codePreviewPattern matches "http://domain/.../{owner}/{repo}/src/commit/{commit}/{filepath}#L10-L20" v.codePreviewPattern = regexp.MustCompile(`https?://\S+/([^\s/]+)/([^\s/]+)/src/commit/([0-9a-f]{7,64})(/\S+)#(L\d+(-L\d+)?)`) - v.tagCleaner = regexp.MustCompile(`<((?:/?\w+/\w+)|(?:/[\w ]+/)|(/?[hH][tT][mM][lL]\b)|(/?[hH][eE][aA][dD]\b))`) + // cleans: "<foo/bar", "<any words/", ("<html", "<head", "<script", "<style", "<?", "<%") + v.tagCleaner = regexp.MustCompile(`(?i)<(/?\w+/\w+|/[\w ]+/|/?(html|head|script|style|%|\?)\b)`) v.nulCleaner = strings.NewReplacer("\000", "") return v }) -// IsFullURLBytes reports whether link fits valid format. -func IsFullURLBytes(link []byte) bool { - return globalVars().fullURLPattern.Match(link) -} - func IsFullURLString(link string) bool { return globalVars().fullURLPattern.MatchString(link) } func IsNonEmptyRelativePath(link string) bool { - return link != "" && !IsFullURLString(link) && link[0] != '/' && link[0] != '?' && link[0] != '#' + return link != "" && !IsFullURLString(link) && link[0] != '?' && link[0] != '#' } // CustomLinkURLSchemes allows for additional schemes to be detected when parsing links within text @@ -116,13 +110,7 @@ func CustomLinkURLSchemes(schemes []string) { if !validScheme.MatchString(s) { continue } - without := false - for _, sna := range xurls.SchemesNoAuthority { - if s == sna { - without = true - break - } - } + without := slices.Contains(xurls.SchemesNoAuthority, s) if without { s += ":" } else { @@ -260,7 +248,7 @@ func postProcess(ctx *RenderContext, procs []processor, input io.Reader, output node, err := html.Parse(io.MultiReader( // prepend "<html><body>" strings.NewReader("<html><body>"), - // Strip out nuls - they're always invalid + // strip out NULLs (they're always invalid), and escape known tags bytes.NewReader(globalVars().tagCleaner.ReplaceAll([]byte(globalVars().nulCleaner.Replace(string(rawHTML))), []byte("<$1"))), // close the tags strings.NewReader("</body></html>"), @@ -316,44 +304,39 @@ func isEmojiNode(node *html.Node) bool { } func visitNode(ctx *RenderContext, procs []processor, node *html.Node) *html.Node { - // Add user-content- to IDs and "#" links if they don't already have them - for idx, attr := range node.Attr { - val := strings.TrimPrefix(attr.Val, "#") - notHasPrefix := !(strings.HasPrefix(val, "user-content-") || globalVars().blackfridayExtRegex.MatchString(val)) - - if attr.Key == "id" && notHasPrefix { - node.Attr[idx].Val = "user-content-" + attr.Val - } - - if attr.Key == "href" && strings.HasPrefix(attr.Val, "#") && notHasPrefix { - node.Attr[idx].Val = "#user-content-" + val - } - } - - switch node.Type { - case html.TextNode: + if node.Type == html.TextNode { for _, proc := range procs { proc(ctx, node) // it might add siblings } + return node.NextSibling + } + if node.Type != html.ElementNode { + return node.NextSibling + } - case html.ElementNode: - if isEmojiNode(node) { - // TextNode emoji will be converted to `<span class="emoji">`, then the next iteration will visit the "span" - // if we don't stop it, it will go into the TextNode again and create an infinite recursion - return node.NextSibling - } else if node.Data == "code" || node.Data == "pre" { - return node.NextSibling // ignore code and pre nodes - } else if node.Data == "img" { - return visitNodeImg(ctx, node) - } else if node.Data == "video" { - return visitNodeVideo(ctx, node) - } else if node.Data == "a" { - procs = emojiProcessors // Restrict text in links to emojis - } - for n := node.FirstChild; n != nil; { - n = visitNode(ctx, procs, n) - } - default: + processNodeAttrID(node) + processFootnoteNode(ctx, node) // FIXME: the footnote processing should be done in the "footnote.go" renderer directly + + if isEmojiNode(node) { + // TextNode emoji will be converted to `<span class="emoji">`, then the next iteration will visit the "span" + // if we don't stop it, it will go into the TextNode again and create an infinite recursion + return node.NextSibling + } else if node.Data == "code" || node.Data == "pre" { + return node.NextSibling // ignore code and pre nodes + } else if node.Data == "img" { + return visitNodeImg(ctx, node) + } else if node.Data == "video" { + return visitNodeVideo(ctx, node) + } + + if node.Data == "a" { + processNodeA(ctx, node) + // only use emoji processors for the content in the "A" tag, + // because the content there is not processable, for example: the content is a commit id or a full URL. + procs = emojiProcessors + } + for n := node.FirstChild; n != nil; { + n = visitNode(ctx, procs, n) } return node.NextSibling } diff --git a/modules/markup/html_commit.go b/modules/markup/html_commit.go index aa1b7d034a..fe7a034967 100644 --- a/modules/markup/html_commit.go +++ b/modules/markup/html_commit.go @@ -43,7 +43,6 @@ func createCodeLink(href, content, class string) *html.Node { code := &html.Node{ Type: html.ElementNode, Data: atom.Code.String(), - Attr: []html.Attribute{{Key: "class", Val: "nohighlight"}}, } code.AppendChild(text) @@ -63,7 +62,7 @@ func anyHashPatternExtract(s string) (ret anyHashPatternResult, ok bool) { // if url ends in '.', it's very likely that it is not part of the actual url but used to finish a sentence. ret.PosEnd-- ret.FullURL = ret.FullURL[:len(ret.FullURL)-1] - for i := 0; i < len(m); i++ { + for i := range m { m[i] = min(m[i], ret.PosEnd) } } @@ -189,7 +188,7 @@ func hashCurrentPatternProcessor(ctx *RenderContext, node *html.Node) { continue } - link := ctx.RenderHelper.ResolveLink(util.URLJoin(ctx.RenderOptions.Metas["user"], ctx.RenderOptions.Metas["repo"], "commit", hash), LinkTypeApp) + link := "/:root/" + util.URLJoin(ctx.RenderOptions.Metas["user"], ctx.RenderOptions.Metas["repo"], "commit", hash) replaceContent(node, m[2], m[3], createCodeLink(link, base.ShortSha(hash), "commit")) start = 0 node = node.NextSibling.NextSibling @@ -205,9 +204,9 @@ func commitCrossReferencePatternProcessor(ctx *RenderContext, node *html.Node) { return } - reftext := ref.Owner + "/" + ref.Name + "@" + base.ShortSha(ref.CommitSha) - linkHref := ctx.RenderHelper.ResolveLink(util.URLJoin(ref.Owner, ref.Name, "commit", ref.CommitSha), LinkTypeApp) - link := createLink(ctx, linkHref, reftext, "commit") + refText := ref.Owner + "/" + ref.Name + "@" + base.ShortSha(ref.CommitSha) + linkHref := "/:root/" + util.URLJoin(ref.Owner, ref.Name, "commit", ref.CommitSha) + link := createLink(ctx, linkHref, refText, "commit") replaceContent(node, ref.RefLocation.Start, ref.RefLocation.End, link) node = node.NextSibling.NextSibling diff --git a/modules/markup/html_email.go b/modules/markup/html_email.go index cbfae8b829..cf18e99d98 100644 --- a/modules/markup/html_email.go +++ b/modules/markup/html_email.go @@ -3,7 +3,11 @@ package markup -import "golang.org/x/net/html" +import ( + "strings" + + "golang.org/x/net/html" +) // emailAddressProcessor replaces raw email addresses with a mailto: link. func emailAddressProcessor(ctx *RenderContext, node *html.Node) { @@ -14,6 +18,14 @@ func emailAddressProcessor(ctx *RenderContext, node *html.Node) { return } + var nextByte byte + if len(node.Data) > m[3] { + nextByte = node.Data[m[3]] + } + if strings.IndexByte(":/", nextByte) != -1 { + // for cases: "git@gitea.com:owner/repo.git", "https://git@gitea.com/owner/repo.git" + return + } mail := node.Data[m[2]:m[3]] replaceContent(node, m[2], m[3], createLink(ctx, "mailto:"+mail, mail, "" /*mailto*/)) node = node.NextSibling.NextSibling diff --git a/modules/markup/html_internal_test.go b/modules/markup/html_internal_test.go index 159d712955..467cc509d0 100644 --- a/modules/markup/html_internal_test.go +++ b/modules/markup/html_internal_test.go @@ -107,7 +107,7 @@ func TestRender_IssueIndexPattern2(t *testing.T) { isExternal := false if marker == "!" { path = "pulls" - prefix = "http://localhost:3000/someUser/someRepo/pulls/" + prefix = "/someUser/someRepo/pulls/" } else { path = "issues" prefix = "https://someurl.com/someUser/someRepo/" @@ -116,7 +116,7 @@ func TestRender_IssueIndexPattern2(t *testing.T) { links := make([]any, len(indices)) for i, index := range indices { - links[i] = numericIssueLink(util.URLJoin(TestRepoURL, path), "ref-issue", index, marker) + links[i] = numericIssueLink(util.URLJoin("/test-owner/test-repo", path), "ref-issue", index, marker) } expectedNil := fmt.Sprintf(expectedFmt, links...) testRenderIssueIndexPattern(t, s, expectedNil, NewTestRenderContext(TestAppURL, localMetas)) @@ -293,13 +293,13 @@ func TestRender_AutoLink(t *testing.T) { // render valid commit URLs tmp := util.URLJoin(TestRepoURL, "commit", "d8a994ef243349f321568f9e36d5c3f444b99cae") - test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code class=\"nohighlight\">d8a994ef24</code></a>") + test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code>d8a994ef24</code></a>") tmp += "#diff-2" - test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code class=\"nohighlight\">d8a994ef24 (diff-2)</code></a>") + test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code>d8a994ef24 (diff-2)</code></a>") // render other commit URLs tmp = "https://external-link.gitea.io/go-gitea/gitea/commit/d8a994ef243349f321568f9e36d5c3f444b99cae#diff-2" - test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code class=\"nohighlight\">d8a994ef24 (diff-2)</code></a>") + test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code>d8a994ef24 (diff-2)</code></a>") } func TestRender_FullIssueURLs(t *testing.T) { @@ -405,10 +405,10 @@ func TestRegExp_anySHA1Pattern(t *testing.T) { if v.CommitID == "" { assert.False(t, ok) } else { - assert.EqualValues(t, strings.TrimSuffix(k, "."), ret.FullURL) - assert.EqualValues(t, v.CommitID, ret.CommitID) - assert.EqualValues(t, v.SubPath, ret.SubPath) - assert.EqualValues(t, v.QueryHash, ret.QueryHash) + assert.Equal(t, strings.TrimSuffix(k, "."), ret.FullURL) + assert.Equal(t, v.CommitID, ret.CommitID) + assert.Equal(t, v.SubPath, ret.SubPath) + assert.Equal(t, v.QueryHash, ret.QueryHash) } } } diff --git a/modules/markup/html_issue.go b/modules/markup/html_issue.go index 7a6f33011a..85bec5db20 100644 --- a/modules/markup/html_issue.go +++ b/modules/markup/html_issue.go @@ -82,7 +82,7 @@ func createIssueLinkContentWithSummary(ctx *RenderContext, linkHref string, ref h, err := DefaultRenderHelperFuncs.RenderRepoIssueIconTitle(ctx, RenderIssueIconTitleOptions{ OwnerName: ref.Owner, RepoName: ref.Name, - LinkHref: linkHref, + LinkHref: ctx.RenderHelper.ResolveLink(linkHref, LinkTypeDefault), IssueIndex: issueIndex, }) if err != nil { @@ -162,7 +162,7 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) { issueOwner := util.Iif(ref.Owner == "", ctx.RenderOptions.Metas["user"], ref.Owner) issueRepo := util.Iif(ref.Owner == "", ctx.RenderOptions.Metas["repo"], ref.Name) issuePath := util.Iif(ref.IsPull, "pulls", "issues") - linkHref := ctx.RenderHelper.ResolveLink(util.URLJoin(issueOwner, issueRepo, issuePath, ref.Issue), LinkTypeApp) + linkHref := "/:root/" + util.URLJoin(issueOwner, issueRepo, issuePath, ref.Issue) // at the moment, only render the issue index in a full line (or simple line) as icon+title // otherwise it would be too noisy for "take #1 as an example" in a sentence diff --git a/modules/markup/html_issue_test.go b/modules/markup/html_issue_test.go index 8d189fbdf6..39cd9dcf6a 100644 --- a/modules/markup/html_issue_test.go +++ b/modules/markup/html_issue_test.go @@ -30,6 +30,7 @@ func TestRender_IssueList(t *testing.T) { rctx := markup.NewTestRenderContext(markup.TestAppURL, map[string]string{ "user": "test-user", "repo": "test-repo", "markupAllowShortIssuePattern": "true", + "footnoteContextId": "12345", }) out, err := markdown.RenderString(rctx, input) require.NoError(t, err) @@ -39,7 +40,7 @@ func TestRender_IssueList(t *testing.T) { t.Run("NormalIssueRef", func(t *testing.T) { test( "#12345", - `<p><a href="http://localhost:3000/test-user/test-repo/issues/12345" class="ref-issue" rel="nofollow">#12345</a></p>`, + `<p><a href="/test-user/test-repo/issues/12345" class="ref-issue" rel="nofollow">#12345</a></p>`, ) }) @@ -56,7 +57,7 @@ func TestRender_IssueList(t *testing.T) { test( "* foo #12345 bar", `<ul> -<li>foo <a href="http://localhost:3000/test-user/test-repo/issues/12345" class="ref-issue" rel="nofollow">#12345</a> bar</li> +<li>foo <a href="/test-user/test-repo/issues/12345" class="ref-issue" rel="nofollow">#12345</a> bar</li> </ul>`, ) }) @@ -69,4 +70,22 @@ func TestRender_IssueList(t *testing.T) { </ul>`, ) }) + + t.Run("IssueFootnote", func(t *testing.T) { + test( + "foo[^1][^2]\n\n[^1]: bar\n[^2]: baz", + `<p>foo<sup id="fnref:user-content-1-12345"><a href="#fn:user-content-1-12345" rel="nofollow">1 </a></sup><sup id="fnref:user-content-2-12345"><a href="#fn:user-content-2-12345" rel="nofollow">2 </a></sup></p> +<div> +<hr/> +<ol> +<li id="fn:user-content-1-12345"> +<p>bar <a href="#fnref:user-content-1-12345" rel="nofollow">↩︎</a></p> +</li> +<li id="fn:user-content-2-12345"> +<p>baz <a href="#fnref:user-content-2-12345" rel="nofollow">↩︎</a></p> +</li> +</ol> +</div>`, + ) + }) } diff --git a/modules/markup/html_link.go b/modules/markup/html_link.go index 0e7a988d36..43faef1681 100644 --- a/modules/markup/html_link.go +++ b/modules/markup/html_link.go @@ -31,8 +31,8 @@ func shortLinkProcessor(ctx *RenderContext, node *html.Node) { // It makes page handling terrible, but we prefer GitHub syntax // And fall back to MediaWiki only when it is obvious from the look // Of text and link contents - sl := strings.Split(content, "|") - for _, v := range sl { + sl := strings.SplitSeq(content, "|") + for v := range sl { if equalPos := strings.IndexByte(v, '='); equalPos == -1 { // There is no equal in this argument; this is a mandatory arg if props["name"] == "" { @@ -125,7 +125,6 @@ func shortLinkProcessor(ctx *RenderContext, node *html.Node) { } } if image { - link = ctx.RenderHelper.ResolveLink(link, LinkTypeMedia) title := props["title"] if title == "" { title = props["alt"] @@ -151,7 +150,6 @@ func shortLinkProcessor(ctx *RenderContext, node *html.Node) { childNode.Attr = childNode.Attr[:2] } } else { - link = ctx.RenderHelper.ResolveLink(link, LinkTypeDefault) childNode.Type = html.TextNode childNode.Data = name } diff --git a/modules/markup/html_mention.go b/modules/markup/html_mention.go index fffa12e7b7..f97c034cf3 100644 --- a/modules/markup/html_mention.go +++ b/modules/markup/html_mention.go @@ -33,7 +33,7 @@ func mentionProcessor(ctx *RenderContext, node *html.Node) { if ok && strings.Contains(mention, "/") { mentionOrgAndTeam := strings.Split(mention, "/") if mentionOrgAndTeam[0][1:] == ctx.RenderOptions.Metas["org"] && strings.Contains(teams, ","+strings.ToLower(mentionOrgAndTeam[1])+",") { - link := ctx.RenderHelper.ResolveLink(util.URLJoin("org", ctx.RenderOptions.Metas["org"], "teams", mentionOrgAndTeam[1]), LinkTypeApp) + link := "/:root/" + util.URLJoin("org", ctx.RenderOptions.Metas["org"], "teams", mentionOrgAndTeam[1]) replaceContent(node, loc.Start, loc.End, createLink(ctx, link, mention, "" /*mention*/)) node = node.NextSibling.NextSibling start = 0 @@ -45,7 +45,7 @@ func mentionProcessor(ctx *RenderContext, node *html.Node) { mentionedUsername := mention[1:] if DefaultRenderHelperFuncs != nil && DefaultRenderHelperFuncs.IsUsernameMentionable(ctx, mentionedUsername) { - link := ctx.RenderHelper.ResolveLink(mentionedUsername, LinkTypeApp) + link := "/:root/" + mentionedUsername replaceContent(node, loc.Start, loc.End, createLink(ctx, link, mention, "" /*mention*/)) node = node.NextSibling.NextSibling start = 0 diff --git a/modules/markup/html_node.go b/modules/markup/html_node.go index 6e8ca67900..4eb78fdd2b 100644 --- a/modules/markup/html_node.go +++ b/modules/markup/html_node.go @@ -4,42 +4,105 @@ package markup import ( + "strings" + "golang.org/x/net/html" ) +func isAnchorIDUserContent(s string) bool { + // blackfridayExtRegex is for blackfriday extensions create IDs like fn:user-content-footnote + // old logic: blackfridayExtRegex = regexp.MustCompile(`[^:]*:user-content-`) + return strings.HasPrefix(s, "user-content-") || strings.Contains(s, ":user-content-") +} + +func isAnchorIDFootnote(s string) bool { + return strings.HasPrefix(s, "fnref:user-content-") || strings.HasPrefix(s, "fn:user-content-") +} + +func isAnchorHrefFootnote(s string) bool { + return strings.HasPrefix(s, "#fnref:user-content-") || strings.HasPrefix(s, "#fn:user-content-") +} + +func processNodeAttrID(node *html.Node) { + // Add user-content- to IDs and "#" links if they don't already have them, + // and convert the link href to a relative link to the host root + for idx, attr := range node.Attr { + if attr.Key == "id" { + if !isAnchorIDUserContent(attr.Val) { + node.Attr[idx].Val = "user-content-" + attr.Val + } + } + } +} + +func processFootnoteNode(ctx *RenderContext, node *html.Node) { + for idx, attr := range node.Attr { + if (attr.Key == "id" && isAnchorIDFootnote(attr.Val)) || + (attr.Key == "href" && isAnchorHrefFootnote(attr.Val)) { + if footnoteContextID := ctx.RenderOptions.Metas["footnoteContextId"]; footnoteContextID != "" { + node.Attr[idx].Val = attr.Val + "-" + footnoteContextID + } + continue + } + } +} + +func processNodeA(ctx *RenderContext, node *html.Node) { + for idx, attr := range node.Attr { + if attr.Key == "href" { + if anchorID, ok := strings.CutPrefix(attr.Val, "#"); ok { + if !isAnchorIDUserContent(attr.Val) { + node.Attr[idx].Val = "#user-content-" + anchorID + } + } else { + node.Attr[idx].Val = ctx.RenderHelper.ResolveLink(attr.Val, LinkTypeDefault) + } + } + } +} + func visitNodeImg(ctx *RenderContext, img *html.Node) (next *html.Node) { next = img.NextSibling - for i, attr := range img.Attr { - if attr.Key != "src" { + attrSrc, hasLazy := "", false + for i, imgAttr := range img.Attr { + hasLazy = hasLazy || imgAttr.Key == "loading" && imgAttr.Val == "lazy" + if imgAttr.Key != "src" { + attrSrc = imgAttr.Val continue } - if IsNonEmptyRelativePath(attr.Val) { - attr.Val = ctx.RenderHelper.ResolveLink(attr.Val, LinkTypeMedia) + imgSrcOrigin := imgAttr.Val + isLinkable := imgSrcOrigin != "" && !strings.HasPrefix(imgSrcOrigin, "data:") - // By default, the "<img>" tag should also be clickable, - // because frontend use `<img>` to paste the re-scaled image into the markdown, - // so it must match the default markdown image behavior. - hasParentAnchor := false - for p := img.Parent; p != nil; p = p.Parent { - if hasParentAnchor = p.Type == html.ElementNode && p.Data == "a"; hasParentAnchor { - break - } - } - if !hasParentAnchor { - imgA := &html.Node{Type: html.ElementNode, Data: "a", Attr: []html.Attribute{ - {Key: "href", Val: attr.Val}, - {Key: "target", Val: "_blank"}, - }} - parent := img.Parent - imgNext := img.NextSibling - parent.RemoveChild(img) - parent.InsertBefore(imgA, imgNext) - imgA.AppendChild(img) + // By default, the "<img>" tag should also be clickable, + // because frontend uses `<img>` to paste the re-scaled image into the Markdown, + // so it must match the default Markdown image behavior. + cnt := 0 + for p := img.Parent; isLinkable && p != nil && cnt < 2; p = p.Parent { + if hasParentAnchor := p.Type == html.ElementNode && p.Data == "a"; hasParentAnchor { + isLinkable = false + break } + cnt++ } - attr.Val = camoHandleLink(attr.Val) - img.Attr[i] = attr + if isLinkable { + wrapper := &html.Node{Type: html.ElementNode, Data: "a", Attr: []html.Attribute{ + {Key: "href", Val: ctx.RenderHelper.ResolveLink(imgSrcOrigin, LinkTypeDefault)}, + {Key: "target", Val: "_blank"}, + }} + parent := img.Parent + imgNext := img.NextSibling + parent.RemoveChild(img) + parent.InsertBefore(wrapper, imgNext) + wrapper.AppendChild(img) + } + + imgAttr.Val = ctx.RenderHelper.ResolveLink(imgSrcOrigin, LinkTypeMedia) + imgAttr.Val = camoHandleLink(imgAttr.Val) + img.Attr[i] = imgAttr + } + if !RenderBehaviorForTesting.DisableAdditionalAttributes && !hasLazy && !strings.HasPrefix(attrSrc, "data:") { + img.Attr = append(img.Attr, html.Attribute{Key: "loading", Val: "lazy"}) } return next } diff --git a/modules/markup/html_test.go b/modules/markup/html_test.go index 6d8f24184b..5fdbf43f7c 100644 --- a/modules/markup/html_test.go +++ b/modules/markup/html_test.go @@ -35,6 +35,7 @@ func TestRender_Commits(t *testing.T) { sha := "65f1bf27bc3bf70f64657658635e66094edbcb4d" repo := markup.TestAppURL + testRepoOwnerName + "/" + testRepoName + "/" commit := util.URLJoin(repo, "commit", sha) + commitPath := "/user13/repo11/commit/" + sha tree := util.URLJoin(repo, "tree", sha, "src") file := util.URLJoin(repo, "commit", sha, "example.txt") @@ -44,9 +45,9 @@ func TestRender_Commits(t *testing.T) { commitCompare := util.URLJoin(repo, "compare", sha+"..."+sha) commitCompareWithHash := commitCompare + "#L2" - test(sha, `<p><a href="`+commit+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) - test(sha[:7], `<p><a href="`+commit[:len(commit)-(40-7)]+`" rel="nofollow"><code>65f1bf2</code></a></p>`) - test(sha[:39], `<p><a href="`+commit[:len(commit)-(40-39)]+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) + test(sha, `<p><a href="`+commitPath+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) + test(sha[:7], `<p><a href="`+commitPath[:len(commitPath)-(40-7)]+`" rel="nofollow"><code>65f1bf2</code></a></p>`) + test(sha[:39], `<p><a href="`+commitPath[:len(commitPath)-(40-39)]+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) test(commit, `<p><a href="`+commit+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) test(tree, `<p><a href="`+tree+`" rel="nofollow"><code>65f1bf27bc/src</code></a></p>`) @@ -57,13 +58,13 @@ func TestRender_Commits(t *testing.T) { test(commitCompare, `<p><a href="`+commitCompare+`" rel="nofollow"><code>65f1bf27bc...65f1bf27bc</code></a></p>`) test(commitCompareWithHash, `<p><a href="`+commitCompareWithHash+`" rel="nofollow"><code>65f1bf27bc...65f1bf27bc (L2)</code></a></p>`) - test("commit "+sha, `<p>commit <a href="`+commit+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) + test("commit "+sha, `<p>commit <a href="`+commitPath+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) test("/home/gitea/"+sha, "<p>/home/gitea/"+sha+"</p>") test("deadbeef", `<p>deadbeef</p>`) test("d27ace93", `<p>d27ace93</p>`) test(sha[:14]+".x", `<p>`+sha[:14]+`.x</p>`) - expected14 := `<a href="` + commit[:len(commit)-(40-14)] + `" rel="nofollow"><code>` + sha[:10] + `</code></a>` + expected14 := `<a href="` + commitPath[:len(commitPath)-(40-14)] + `" rel="nofollow"><code>` + sha[:10] + `</code></a>` test(sha[:14]+".", `<p>`+expected14+`.</p>`) test(sha[:14]+",", `<p>`+expected14+`,</p>`) test("["+sha[:14]+"]", `<p>[`+expected14+`]</p>`) @@ -80,10 +81,10 @@ func TestRender_CrossReferences(t *testing.T) { test( "test-owner/test-repo#12345", - `<p><a href="`+util.URLJoin(markup.TestAppURL, "test-owner", "test-repo", "issues", "12345")+`" class="ref-issue" rel="nofollow">test-owner/test-repo#12345</a></p>`) + `<p><a href="/test-owner/test-repo/issues/12345" class="ref-issue" rel="nofollow">test-owner/test-repo#12345</a></p>`) test( "go-gitea/gitea#12345", - `<p><a href="`+util.URLJoin(markup.TestAppURL, "go-gitea", "gitea", "issues", "12345")+`" class="ref-issue" rel="nofollow">go-gitea/gitea#12345</a></p>`) + `<p><a href="/go-gitea/gitea/issues/12345" class="ref-issue" rel="nofollow">go-gitea/gitea#12345</a></p>`) test( "/home/gitea/go-gitea/gitea#12345", `<p>/home/gitea/go-gitea/gitea#12345</p>`) @@ -224,10 +225,10 @@ func TestRender_email(t *testing.T) { test := func(input, expected string) { res, err := markup.RenderString(markup.NewTestRenderContext().WithRelativePath("a.md"), input) assert.NoError(t, err) - assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(res)) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(res), "input: %s", input) } - // Text that should be turned into email link + // Text that should be turned into email link test( "info@gitea.com", `<p><a href="mailto:info@gitea.com" rel="nofollow">info@gitea.com</a></p>`) @@ -259,28 +260,48 @@ func TestRender_email(t *testing.T) { <a href="mailto:j.doe@example.com" rel="nofollow">j.doe@example.com</a>? <a href="mailto:j.doe@example.com" rel="nofollow">j.doe@example.com</a>!</p>`) + // match GitHub behavior + test("email@domain@domain.com", `<p>email@<a href="mailto:domain@domain.com" rel="nofollow">domain@domain.com</a></p>`) + + // match GitHub behavior + test(`"info@gitea.com"`, `<p>"<a href="mailto:info@gitea.com" rel="nofollow">info@gitea.com</a>"</p>`) + // Test that should *not* be turned into email links test( - "\"info@gitea.com\"", - `<p>"info@gitea.com"</p>`) - test( "/home/gitea/mailstore/info@gitea/com", `<p>/home/gitea/mailstore/info@gitea/com</p>`) test( "git@try.gitea.io:go-gitea/gitea.git", `<p>git@try.gitea.io:go-gitea/gitea.git</p>`) test( + "https://foo:bar@gitea.io", + `<p><a href="https://foo:bar@gitea.io" rel="nofollow">https://foo:bar@gitea.io</a></p>`) + test( "gitea@3", `<p>gitea@3</p>`) test( "gitea@gmail.c", `<p>gitea@gmail.c</p>`) test( - "email@domain@domain.com", - `<p>email@domain@domain.com</p>`) - test( "email@domain..com", `<p>email@domain..com</p>`) + + cases := []struct { + input, expected string + }{ + // match GitHub behavior + {"?a@d.zz", `<p>?<a href="mailto:a@d.zz" rel="nofollow">a@d.zz</a></p>`}, + {"*a@d.zz", `<p>*<a href="mailto:a@d.zz" rel="nofollow">a@d.zz</a></p>`}, + {"~a@d.zz", `<p>~<a href="mailto:a@d.zz" rel="nofollow">a@d.zz</a></p>`}, + + // the following cases don't match GitHub behavior, but they are valid email addresses ... + // maybe we should reduce the candidate characters for the "name" part in the future + {"a*a@d.zz", `<p><a href="mailto:a*a@d.zz" rel="nofollow">a*a@d.zz</a></p>`}, + {"a~a@d.zz", `<p><a href="mailto:a~a@d.zz" rel="nofollow">a~a@d.zz</a></p>`}, + } + for _, c := range cases { + test(c.input, c.expected) + } } func TestRender_emoji(t *testing.T) { @@ -468,7 +489,7 @@ func Test_ParseClusterFuzz(t *testing.T) { assert.NotContains(t, res.String(), "<html") } -func TestPostProcess_RenderDocument(t *testing.T) { +func TestPostProcess(t *testing.T) { setting.StaticURLPrefix = markup.TestAppURL // can't run standalone defer testModule.MockVariableValue(&markup.RenderBehaviorForTesting.DisableAdditionalAttributes, true)() @@ -479,7 +500,7 @@ func TestPostProcess_RenderDocument(t *testing.T) { assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(res.String())) } - // Issue index shouldn't be post processing in a document. + // Issue index shouldn't be post-processing in a document. test( "#1", "#1") @@ -487,9 +508,9 @@ func TestPostProcess_RenderDocument(t *testing.T) { // But cross-referenced issue index should work. test( "go-gitea/gitea#12345", - `<a href="`+util.URLJoin(markup.TestAppURL, "go-gitea", "gitea", "issues", "12345")+`" class="ref-issue">go-gitea/gitea#12345</a>`) + `<a href="/go-gitea/gitea/issues/12345" class="ref-issue">go-gitea/gitea#12345</a>`) - // Test that other post processing still works. + // Test that other post-processing still works. test( ":gitea:", `<span class="emoji" aria-label="gitea"><img alt=":gitea:" src="`+setting.StaticURLPrefix+`/assets/img/emoji/gitea.png"/></span>`) @@ -498,6 +519,16 @@ func TestPostProcess_RenderDocument(t *testing.T) { `Some text with <span class="emoji" aria-label="grinning face with smiling eyes">😄</span> in the middle`) test("http://localhost:3000/person/repo/issues/4#issuecomment-1234", `<a href="http://localhost:3000/person/repo/issues/4#issuecomment-1234" class="ref-issue">person/repo#4 (comment)</a>`) + + // special tags, GitHub's behavior, and for unclosed tags, output as text content as much as possible + test("<script>a", `<script>a`) + test("<script>a</script>", `<script>a</script>`) + test("<STYLE>a", `<STYLE>a`) + test("<style>a</STYLE>", `<style>a</STYLE>`) + + // other special tags, our special behavior + test("<?php\nfoo", "<?php\nfoo") + test("<%asp\nfoo", "<%asp\nfoo") } func TestIssue16020(t *testing.T) { @@ -522,7 +553,7 @@ func BenchmarkEmojiPostprocess(b *testing.B) { data += data } b.ResetTimer() - for i := 0; i < b.N; i++ { + for b.Loop() { var res strings.Builder err := markup.PostProcessDefault(markup.NewTestRenderContext(localMetas), strings.NewReader(data), &res) assert.NoError(b, err) @@ -543,7 +574,7 @@ func TestIssue18471(t *testing.T) { err := markup.PostProcessDefault(markup.NewTestRenderContext(localMetas), strings.NewReader(data), &res) assert.NoError(t, err) - assert.Equal(t, `<a href="http://domain/org/repo/compare/783b039...da951ce" class="compare"><code class="nohighlight">783b039...da951ce</code></a>`, res.String()) + assert.Equal(t, `<a href="http://domain/org/repo/compare/783b039...da951ce" class="compare"><code>783b039...da951ce</code></a>`, res.String()) } func TestIsFullURL(t *testing.T) { diff --git a/modules/markup/internal/internal_test.go b/modules/markup/internal/internal_test.go index 98ff3bc079..590bcbb67f 100644 --- a/modules/markup/internal/internal_test.go +++ b/modules/markup/internal/internal_test.go @@ -35,7 +35,7 @@ func TestRenderInternal(t *testing.T) { assert.EqualValues(t, c.protected, protected) _, _ = io.WriteString(in, string(protected)) _ = in.Close() - assert.EqualValues(t, c.recovered, out.String()) + assert.Equal(t, c.recovered, out.String()) } var r1, r2 RenderInternal @@ -44,11 +44,11 @@ func TestRenderInternal(t *testing.T) { _ = r1.init("sec", nil) protected = r1.ProtectSafeAttrs(`<div class="test"></div>`) assert.EqualValues(t, `<div data-attr-class="sec:test"></div>`, protected) - assert.EqualValues(t, "data-attr-class", r1.SafeAttr("class")) - assert.EqualValues(t, "sec:val", r1.SafeValue("val")) + assert.Equal(t, "data-attr-class", r1.SafeAttr("class")) + assert.Equal(t, "sec:val", r1.SafeValue("val")) recovered, ok := r1.RecoverProtectedValue("sec:val") assert.True(t, ok) - assert.EqualValues(t, "val", recovered) + assert.Equal(t, "val", recovered) recovered, ok = r1.RecoverProtectedValue("other:val") assert.False(t, ok) assert.Empty(t, recovered) @@ -57,5 +57,5 @@ func TestRenderInternal(t *testing.T) { in2 := r2.init("sec-other", out2) _, _ = io.WriteString(in2, string(protected)) _ = in2.Close() - assert.EqualValues(t, `<div data-attr-class="sec:test"></div>`, out2.String(), "different secureID should not recover the value") + assert.Equal(t, `<div data-attr-class="sec:test"></div>`, out2.String(), "different secureID should not recover the value") } diff --git a/modules/markup/markdown/ast.go b/modules/markup/markdown/ast.go index ca165b1ba0..f29f883734 100644 --- a/modules/markup/markdown/ast.go +++ b/modules/markup/markdown/ast.go @@ -4,6 +4,7 @@ package markdown import ( + "html/template" "strconv" "github.com/yuin/goldmark/ast" @@ -29,9 +30,7 @@ func (n *Details) Kind() ast.NodeKind { // NewDetails returns a new Paragraph node. func NewDetails() *Details { - return &Details{ - BaseBlock: ast.BaseBlock{}, - } + return &Details{} } // Summary is a block that contains the summary of details block @@ -54,9 +53,7 @@ func (n *Summary) Kind() ast.NodeKind { // NewSummary returns a new Summary node. func NewSummary() *Summary { - return &Summary{ - BaseBlock: ast.BaseBlock{}, - } + return &Summary{} } // TaskCheckBoxListItem is a block that represents a list item of a markdown block with a checkbox @@ -95,29 +92,6 @@ type Icon struct { Name []byte } -// Dump implements Node.Dump . -func (n *Icon) Dump(source []byte, level int) { - m := map[string]string{} - m["Name"] = string(n.Name) - ast.DumpHelper(n, source, level, m, nil) -} - -// KindIcon is the NodeKind for Icon -var KindIcon = ast.NewNodeKind("Icon") - -// Kind implements Node.Kind. -func (n *Icon) Kind() ast.NodeKind { - return KindIcon -} - -// NewIcon returns a new Paragraph node. -func NewIcon(name string) *Icon { - return &Icon{ - BaseInline: ast.BaseInline{}, - Name: []byte(name), - } -} - // ColorPreview is an inline for a color preview type ColorPreview struct { ast.BaseInline @@ -175,3 +149,24 @@ func NewAttention(attentionType string) *Attention { AttentionType: attentionType, } } + +var KindRawHTML = ast.NewNodeKind("RawHTML") + +type RawHTML struct { + ast.BaseBlock + rawHTML template.HTML +} + +func (n *RawHTML) Dump(source []byte, level int) { + m := map[string]string{} + m["RawHTML"] = string(n.rawHTML) + ast.DumpHelper(n, source, level, m, nil) +} + +func (n *RawHTML) Kind() ast.NodeKind { + return KindRawHTML +} + +func NewRawHTML(rawHTML template.HTML) *RawHTML { + return &RawHTML{rawHTML: rawHTML} +} diff --git a/modules/markup/markdown/convertyaml.go b/modules/markup/markdown/convertyaml.go index 1675b68be2..04664a9c1d 100644 --- a/modules/markup/markdown/convertyaml.go +++ b/modules/markup/markdown/convertyaml.go @@ -4,23 +4,22 @@ package markdown import ( + "strings" + + "code.gitea.io/gitea/modules/htmlutil" + "code.gitea.io/gitea/modules/svg" + "github.com/yuin/goldmark/ast" east "github.com/yuin/goldmark/extension/ast" "gopkg.in/yaml.v3" ) func nodeToTable(meta *yaml.Node) ast.Node { - for { - if meta == nil { - return nil - } - switch meta.Kind { - case yaml.DocumentNode: - meta = meta.Content[0] - continue - default: - } - break + for meta != nil && meta.Kind == yaml.DocumentNode { + meta = meta.Content[0] + } + if meta == nil { + return nil } switch meta.Kind { case yaml.MappingNode: @@ -72,12 +71,28 @@ func sequenceNodeToTable(meta *yaml.Node) ast.Node { return table } -func nodeToDetails(meta *yaml.Node, icon string) ast.Node { +func nodeToDetails(g *ASTTransformer, meta *yaml.Node) ast.Node { + for meta != nil && meta.Kind == yaml.DocumentNode { + meta = meta.Content[0] + } + if meta == nil { + return nil + } + if meta.Kind != yaml.MappingNode { + return nil + } + var keys []string + for i := 0; i < len(meta.Content); i += 2 { + if meta.Content[i].Kind == yaml.ScalarNode { + keys = append(keys, meta.Content[i].Value) + } + } details := NewDetails() + details.SetAttributeString(g.renderInternal.SafeAttr("class"), g.renderInternal.SafeValue("frontmatter-content")) summary := NewSummary() - summary.AppendChild(summary, NewIcon(icon)) + summaryInnerHTML := htmlutil.HTMLFormat("%s %s", svg.RenderHTML("octicon-table", 12), strings.Join(keys, ", ")) + summary.AppendChild(summary, NewRawHTML(summaryInnerHTML)) details.AppendChild(details, summary) details.AppendChild(details, nodeToTable(meta)) - return details } diff --git a/modules/markup/markdown/goldmark.go b/modules/markup/markdown/goldmark.go index 69c2a96ff1..b28fa9824e 100644 --- a/modules/markup/markdown/goldmark.go +++ b/modules/markup/markdown/goldmark.go @@ -5,14 +5,10 @@ package markdown import ( "fmt" - "regexp" - "strings" - "sync" "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/internal" - "code.gitea.io/gitea/modules/setting" "github.com/yuin/goldmark/ast" east "github.com/yuin/goldmark/extension/ast" @@ -51,7 +47,7 @@ func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc pa tocList := make([]Header, 0, 20) if rc.yamlNode != nil { - metaNode := rc.toMetaNode() + metaNode := rc.toMetaNode(g) if metaNode != nil { node.InsertBefore(node, firstChild, metaNode) } @@ -68,23 +64,12 @@ func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc pa g.transformHeading(ctx, v, reader, &tocList) case *ast.Paragraph: g.applyElementDir(v) - case *ast.Image: - g.transformImage(ctx, v) - case *ast.Link: - g.transformLink(ctx, v) case *ast.List: g.transformList(ctx, v, rc) case *ast.Text: if v.SoftLineBreak() && !v.HardLineBreak() { - // TODO: this was a quite unclear part, old code: `if metas["mode"] != "document" { use comment link break setting }` - // many places render non-comment contents with no mode=document, then these contents also use comment's hard line break setting - // especially in many tests. - markdownLineBreakStyle := ctx.RenderOptions.Metas["markdownLineBreakStyle"] - if markdownLineBreakStyle == "comment" { - v.SetHardLineBreak(setting.Markdown.EnableHardLineBreakInComments) - } else if markdownLineBreakStyle == "document" { - v.SetHardLineBreak(setting.Markdown.EnableHardLineBreakInDocuments) - } + newLineHardBreak := ctx.RenderOptions.Metas["markdownNewLineHardBreak"] == "true" + v.SetHardLineBreak(newLineHardBreak) } case *ast.CodeSpan: g.transformCodeSpan(ctx, v, reader) @@ -111,11 +96,6 @@ func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc pa } } -// it is copied from old code, which is quite doubtful whether it is correct -var reValidIconName = sync.OnceValue(func() *regexp.Regexp { - return regexp.MustCompile(`^[-\w]+$`) // old: regexp.MustCompile("^[a-z ]+$") -}) - // NewHTMLRenderer creates a HTMLRenderer to render in the gitea form. func NewHTMLRenderer(renderInternal *internal.RenderInternal, opts ...html.Option) renderer.NodeRenderer { r := &HTMLRenderer{ @@ -140,11 +120,11 @@ func (r *HTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { reg.Register(ast.KindDocument, r.renderDocument) reg.Register(KindDetails, r.renderDetails) reg.Register(KindSummary, r.renderSummary) - reg.Register(KindIcon, r.renderIcon) reg.Register(ast.KindCodeSpan, r.renderCodeSpan) reg.Register(KindAttention, r.renderAttention) reg.Register(KindTaskCheckBoxListItem, r.renderTaskCheckBoxListItem) reg.Register(east.KindTaskCheckBox, r.renderTaskCheckBox) + reg.Register(KindRawHTML, r.renderRawHTML) } func (r *HTMLRenderer) renderDocument(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { @@ -155,7 +135,7 @@ func (r *HTMLRenderer) renderDocument(w util.BufWriter, source []byte, node ast. if entering { _, err = w.WriteString("<div") if err == nil { - _, err = w.WriteString(fmt.Sprintf(` lang=%q`, val)) + _, err = fmt.Fprintf(w, ` lang=%q`, val) } if err == nil { _, err = w.WriteRune('>') @@ -206,30 +186,14 @@ func (r *HTMLRenderer) renderSummary(w util.BufWriter, source []byte, node ast.N return ast.WalkContinue, nil } -func (r *HTMLRenderer) renderIcon(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { +func (r *HTMLRenderer) renderRawHTML(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { if !entering { return ast.WalkContinue, nil } - - n := node.(*Icon) - - name := strings.TrimSpace(strings.ToLower(string(n.Name))) - - if len(name) == 0 { - // skip this - return ast.WalkContinue, nil - } - - if !reValidIconName().MatchString(name) { - // skip this - return ast.WalkContinue, nil - } - - // FIXME: the "icon xxx" is from Fomantic UI, it's really questionable whether it still works correctly - err := r.renderInternal.FormatWithSafeAttrs(w, `<i class="icon %s"></i>`, name) + n := node.(*RawHTML) + _, err := w.WriteString(string(r.renderInternal.ProtectSafeAttrs(n.rawHTML))) if err != nil { return ast.WalkStop, err } - return ast.WalkContinue, nil } diff --git a/modules/markup/markdown/markdown.go b/modules/markup/markdown/markdown.go index b5fffccdb9..3b788432ba 100644 --- a/modules/markup/markdown/markdown.go +++ b/modules/markup/markdown/markdown.go @@ -5,7 +5,7 @@ package markdown import ( - "fmt" + "errors" "html/template" "io" "strings" @@ -48,7 +48,7 @@ func (l *limitWriter) Write(data []byte) (int, error) { if err != nil { return n, err } - return n, fmt.Errorf("rendered content too large - truncating render") + return n, errors.New("rendered content too large - truncating render") } n, err := l.w.Write(data) l.sum += int64(n) @@ -86,20 +86,15 @@ func (r *GlodmarkRender) highlightingRenderer(w util.BufWriter, c highlighting.C preClasses += " is-loading" } - err := r.ctx.RenderInternal.FormatWithSafeAttrs(w, `<pre class="%s">`, preClasses) - if err != nil { - return - } - // include language-x class as part of commonmark spec, "chroma" class is used to highlight the code // the "display" class is used by "js/markup/math.ts" to render the code element as a block // the "math.ts" strictly depends on the structure: <pre class="code-block is-loading"><code class="language-math display">...</code></pre> - err = r.ctx.RenderInternal.FormatWithSafeAttrs(w, `<code class="chroma language-%s display">`, languageStr) + err := r.ctx.RenderInternal.FormatWithSafeAttrs(w, `<div class="code-block-container code-overflow-scroll"><pre class="%s"><code class="chroma language-%s display">`, preClasses, languageStr) if err != nil { return } } else { - _, err := w.WriteString("</code></pre>") + _, err := w.WriteString("</code></pre></div>") if err != nil { return } @@ -126,11 +121,11 @@ func SpecializedMarkdown(ctx *markup.RenderContext) *GlodmarkRender { highlighting.WithWrapperRenderer(r.highlightingRenderer), ), math.NewExtension(&ctx.RenderInternal, math.Options{ - Enabled: setting.Markdown.EnableMath, - ParseDollarInline: true, - ParseDollarBlock: true, - ParseSquareBlock: true, // TODO: this is a bad syntax "\[ ... \]", it conflicts with normal markdown escaping, it should be deprecated in the future (by some config options) - // ParseBracketInline: true, // TODO: this is also a bad syntax "\( ... \)", it also conflicts, it should be deprecated in the future + Enabled: setting.Markdown.EnableMath, + ParseInlineDollar: setting.Markdown.MathCodeBlockOptions.ParseInlineDollar, + ParseInlineParentheses: setting.Markdown.MathCodeBlockOptions.ParseInlineParentheses, // this is a bad syntax "\( ... \)", it conflicts with normal markdown escaping + ParseBlockDollar: setting.Markdown.MathCodeBlockOptions.ParseBlockDollar, + ParseBlockSquareBrackets: setting.Markdown.MathCodeBlockOptions.ParseBlockSquareBrackets, // this is a bad syntax "\[ ... \]", it conflicts with normal markdown escaping }), meta.Meta, ), @@ -159,6 +154,14 @@ func render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error limit: setting.UI.MaxDisplayFileSize * 3, } + // FIXME: Don't read all to memory, but goldmark doesn't support + buf, err := io.ReadAll(input) + if err != nil { + log.Error("Unable to ReadAll: %v", err) + return err + } + buf = giteautil.NormalizeEOL(buf) + // FIXME: should we include a timeout to abort the renderer if it takes too long? defer func() { err := recover() @@ -166,35 +169,20 @@ func render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error return } - log.Warn("Unable to render markdown due to panic in goldmark: %v", err) - if (!setting.IsProd && !setting.IsInTesting) || log.IsDebug() { - log.Error("Panic in markdown: %v\n%s", err, log.Stack(2)) - } + log.Error("Panic in markdown: %v\n%s", err, log.Stack(2)) + escapedHTML := template.HTMLEscapeString(giteautil.UnsafeBytesToString(buf)) + _, _ = output.Write(giteautil.UnsafeStringToBytes(escapedHTML)) }() - // FIXME: Don't read all to memory, but goldmark doesn't support pc := newParserContext(ctx) - buf, err := io.ReadAll(input) - if err != nil { - log.Error("Unable to ReadAll: %v", err) - return err - } - buf = giteautil.NormalizeEOL(buf) // Preserve original length. bufWithMetadataLength := len(buf) - rc := &RenderConfig{ - Meta: markup.RenderMetaAsDetails, - Icon: "table", - Lang: "", - } + rc := &RenderConfig{Meta: markup.RenderMetaAsDetails} buf, _ = ExtractMetadataBytes(buf, rc) - metaLength := bufWithMetadataLength - len(buf) - if metaLength < 0 { - metaLength = 0 - } + metaLength := max(bufWithMetadataLength-len(buf), 0) rc.metaLength = metaLength pc.Set(renderConfigKey, rc) diff --git a/modules/markup/markdown/markdown_attention_test.go b/modules/markup/markdown/markdown_attention_test.go index f6ec775b2c..7b54653ec0 100644 --- a/modules/markup/markdown/markdown_attention_test.go +++ b/modules/markup/markdown/markdown_attention_test.go @@ -23,6 +23,11 @@ func TestAttention(t *testing.T) { defer svg.MockIcon("octicon-alert")() defer svg.MockIcon("octicon-stop")() + test := func(input, expected string) { + result, err := markdown.RenderString(markup.NewTestRenderContext(), input) + assert.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(result))) + } renderAttention := func(attention, icon string) string { tmpl := `<blockquote class="attention-header attention-{attention}"><p><svg class="attention-icon attention-{attention} svg {icon}" width="16" height="16"></svg><strong class="attention-{attention}">{Attention}</strong></p>` tmpl = strings.ReplaceAll(tmpl, "{attention}", attention) @@ -31,12 +36,6 @@ func TestAttention(t *testing.T) { return tmpl } - test := func(input, expected string) { - result, err := markdown.RenderString(markup.NewTestRenderContext(), input) - assert.NoError(t, err) - assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(result))) - } - test(` > [!NOTE] > text @@ -53,4 +52,7 @@ func TestAttention(t *testing.T) { // legacy GitHub style test(`> **warning**`, renderAttention("warning", "octicon-alert")+"\n</blockquote>") + + // edge case (it used to cause panic) + test(">\ntext", "<blockquote>\n</blockquote>\n<p>text</p>") } diff --git a/modules/markup/markdown/markdown_benchmark_test.go b/modules/markup/markdown/markdown_benchmark_test.go index 0f7e3eea6f..e08612f064 100644 --- a/modules/markup/markdown/markdown_benchmark_test.go +++ b/modules/markup/markdown/markdown_benchmark_test.go @@ -12,14 +12,14 @@ import ( func BenchmarkSpecializedMarkdown(b *testing.B) { // 240856 4719 ns/op - for i := 0; i < b.N; i++ { + for b.Loop() { markdown.SpecializedMarkdown(&markup.RenderContext{}) } } func BenchmarkMarkdownRender(b *testing.B) { // 23202 50840 ns/op - for i := 0; i < b.N; i++ { + for b.Loop() { _, _ = markdown.RenderString(markup.NewTestRenderContext(), "https://example.com\n- a\n- b\n") } } diff --git a/modules/markup/markdown/markdown_math_test.go b/modules/markup/markdown/markdown_math_test.go index 813f050965..a75f18d36a 100644 --- a/modules/markup/markdown/markdown_math_test.go +++ b/modules/markup/markdown/markdown_math_test.go @@ -8,6 +8,8 @@ import ( "testing" "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" "github.com/stretchr/testify/assert" ) @@ -15,6 +17,7 @@ import ( const nl = "\n" func TestMathRender(t *testing.T) { + setting.Markdown.MathCodeBlockOptions = setting.MarkdownMathCodeBlockOptions{ParseInlineDollar: true, ParseInlineParentheses: true} testcases := []struct { testcase string expected string @@ -69,7 +72,7 @@ func TestMathRender(t *testing.T) { }, { "$$a$$", - `<code class="language-math display">a</code>` + nl, + `<p><code class="language-math">a</code></p>` + nl, }, { "$$a$$ test", @@ -111,6 +114,7 @@ func TestMathRender(t *testing.T) { } func TestMathRenderBlockIndent(t *testing.T) { + setting.Markdown.MathCodeBlockOptions = setting.MarkdownMathCodeBlockOptions{ParseBlockDollar: true, ParseBlockSquareBrackets: true} testcases := []struct { name string testcase string @@ -243,3 +247,64 @@ x }) } } + +func TestMathRenderOptions(t *testing.T) { + setting.Markdown.MathCodeBlockOptions = setting.MarkdownMathCodeBlockOptions{} + defer test.MockVariableValue(&setting.Markdown.MathCodeBlockOptions) + test := func(t *testing.T, expected, input string) { + res, err := RenderString(markup.NewTestRenderContext(), input) + assert.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(res)), "input: %s", input) + } + + // default (non-conflict) inline syntax + test(t, `<p><code class="language-math">a</code></p>`, "$`a`$") + + // ParseInlineDollar + test(t, `<p>$a$</p>`, `$a$`) + setting.Markdown.MathCodeBlockOptions.ParseInlineDollar = true + test(t, `<p><code class="language-math">a</code></p>`, `$a$`) + + // ParseInlineParentheses + test(t, `<p>(a)</p>`, `\(a\)`) + setting.Markdown.MathCodeBlockOptions.ParseInlineParentheses = true + test(t, `<p><code class="language-math">a</code></p>`, `\(a\)`) + + // ParseBlockDollar + test(t, `<p>$$ +a +$$</p> +`, ` +$$ +a +$$ +`) + setting.Markdown.MathCodeBlockOptions.ParseBlockDollar = true + test(t, `<pre class="code-block is-loading"><code class="language-math display"> +a +</code></pre> +`, ` +$$ +a +$$ +`) + + // ParseBlockSquareBrackets + test(t, `<p>[ +a +]</p> +`, ` +\[ +a +\] +`) + setting.Markdown.MathCodeBlockOptions.ParseBlockSquareBrackets = true + test(t, `<pre class="code-block is-loading"><code class="language-math display"> +a +</code></pre> +`, ` +\[ +a +\] +`) +} diff --git a/modules/markup/markdown/markdown_test.go b/modules/markup/markdown/markdown_test.go index 7a09be8665..4eb01bcc2d 100644 --- a/modules/markup/markdown/markdown_test.go +++ b/modules/markup/markdown/markdown_test.go @@ -47,7 +47,7 @@ func TestRender_StandardLinks(t *testing.T) { func TestRender_Images(t *testing.T) { setting.AppURL = AppURL - test := func(input, expected string) { + render := func(input, expected string) { buffer, err := markdown.RenderString(markup.NewTestRenderContext(FullURL), input) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer))) @@ -59,27 +59,32 @@ func TestRender_Images(t *testing.T) { result := util.URLJoin(FullURL, url) // hint: With Markdown v2.5.2, there is a new syntax: [link](URL){:target="_blank"} , but we do not support it now - test( + render( "!["+title+"]("+url+")", `<p><a href="`+result+`" target="_blank" rel="nofollow noopener"><img src="`+result+`" alt="`+title+`"/></a></p>`) - test( + render( "[["+title+"|"+url+"]]", `<p><a href="`+result+`" rel="nofollow"><img src="`+result+`" title="`+title+`" alt="`+title+`"/></a></p>`) - test( + render( "[!["+title+"]("+url+")]("+href+")", `<p><a href="`+href+`" rel="nofollow"><img src="`+result+`" alt="`+title+`"/></a></p>`) - test( + render( "!["+title+"]("+url+")", `<p><a href="`+result+`" target="_blank" rel="nofollow noopener"><img src="`+result+`" alt="`+title+`"/></a></p>`) - test( + render( "[["+title+"|"+url+"]]", `<p><a href="`+result+`" rel="nofollow"><img src="`+result+`" title="`+title+`" alt="`+title+`"/></a></p>`) - test( + render( "[!["+title+"]("+url+")]("+href+")", `<p><a href="`+href+`" rel="nofollow"><img src="`+result+`" alt="`+title+`"/></a></p>`) + + defer test.MockVariableValue(&markup.RenderBehaviorForTesting.DisableAdditionalAttributes, false)() + render( + "<a><img src='a.jpg'></a>", // by the way, empty "a" tag will be removed + `<p dir="auto"><img src="http://localhost:3000/user13/repo11/a.jpg" loading="lazy"/></p>`) } func TestTotal_RenderString(t *testing.T) { @@ -223,7 +228,7 @@ This PR has been generated by [Renovate Bot](https://github.com/renovatebot/reno <dd>This is another definition of the second term.</dd> </dl> <h3 id="user-content-footnotes">Footnotes</h3> -<p>Here is a simple footnote,<sup id="fnref:user-content-1"><a href="#fn:user-content-1" rel="nofollow">1</a></sup> and here is a longer one.<sup id="fnref:user-content-bignote"><a href="#fn:user-content-bignote" rel="nofollow">2</a></sup></p> +<p>Here is a simple footnote,<sup id="fnref:user-content-1"><a href="#fn:user-content-1" rel="nofollow">1 </a></sup> and here is a longer one.<sup id="fnref:user-content-bignote"><a href="#fn:user-content-bignote" rel="nofollow">2 </a></sup></p> <div> <hr/> <ol> @@ -252,7 +257,7 @@ This PR has been generated by [Renovate Bot](https://github.com/renovatebot/reno return username == "r-lyeh" }, }) - for i := 0; i < len(sameCases); i++ { + for i := range sameCases { line, err := markdown.RenderString(markup.NewTestRenderContext(localMetas), sameCases[i]) assert.NoError(t, err) assert.Equal(t, testAnswers[i], string(line)) @@ -308,12 +313,12 @@ func TestRenderSiblingImages_Issue12925(t *testing.T) { testcase := `  ` - expected := `<p><a href="/image1" target="_blank" rel="nofollow noopener"><img src="/image1" alt="image1"></a> -<a href="/image2" target="_blank" rel="nofollow noopener"><img src="/image2" alt="image2"></a></p> + expected := `<p><a href="/image1" target="_blank" rel="nofollow noopener"><img src="/image1" alt="image1"/></a> +<a href="/image2" target="_blank" rel="nofollow noopener"><img src="/image2" alt="image2"/></a></p> ` - res, err := markdown.RenderRawString(markup.NewTestRenderContext(), testcase) + res, err := markdown.RenderString(markup.NewTestRenderContext(), testcase) assert.NoError(t, err) - assert.Equal(t, expected, res) + assert.Equal(t, expected, string(res)) } func TestRenderEmojiInLinks_Issue12331(t *testing.T) { @@ -383,18 +388,74 @@ func TestColorPreview(t *testing.T) { } } -func TestTaskList(t *testing.T) { +func TestMarkdownFrontmatter(t *testing.T) { testcases := []struct { - testcase string + name string + input string expected string }{ { + "MapInFrontmatter", + `--- +key1: val1 +key2: val2 +--- +test +`, + `<details class="frontmatter-content"><summary><span>octicon-table(12/)</span> key1, key2</summary><table> +<thead> +<tr> +<th>key1</th> +<th>key2</th> +</tr> +</thead> +<tbody> +<tr> +<td>val1</td> +<td>val2</td> +</tr> +</tbody> +</table> +</details><p>test</p> +`, + }, + + { + "ListInFrontmatter", + `--- +- item1 +- item2 +--- +test +`, + `- item1 +- item2 + +<p>test</p> +`, + }, + + { + "StringInFrontmatter", + `--- +anything +--- +test +`, + `anything + +<p>test</p> +`, + }, + + { // data-source-position should take into account YAML frontmatter. + "ListAfterFrontmatter", `--- foo: bar --- - [ ] task 1`, - `<details><summary><i class="icon table"></i></summary><table> + `<details class="frontmatter-content"><summary><span>octicon-table(12/)</span> foo</summary><table> <thead> <tr> <th>foo</th> @@ -414,9 +475,9 @@ foo: bar } for _, test := range testcases { - res, err := markdown.RenderString(markup.NewTestRenderContext(), test.testcase) - assert.NoError(t, err, "Unexpected error in testcase: %q", test.testcase) - assert.Equal(t, template.HTML(test.expected), res, "Unexpected result in testcase %q", test.testcase) + res, err := markdown.RenderString(markup.NewTestRenderContext(), test.input) + assert.NoError(t, err, "Unexpected error in testcase: %q", test.name) + assert.Equal(t, test.expected, string(res), "Unexpected result in testcase %q", test.name) } } @@ -473,3 +534,16 @@ space</p> assert.NoError(t, err) assert.Equal(t, expected, string(result)) } + +func TestMarkdownLink(t *testing.T) { + defer test.MockVariableValue(&markup.RenderBehaviorForTesting.DisableAdditionalAttributes, true)() + input := `<a href=foo>link1</a> +<a href='/foo'>link2</a> +<a href="#foo">link3</a>` + result, err := markdown.RenderString(markup.NewTestRenderContext("/base", localMetas), input) + assert.NoError(t, err) + assert.Equal(t, `<p><a href="/base/foo" rel="nofollow">link1</a> +<a href="/base/foo" rel="nofollow">link2</a> +<a href="#user-content-foo" rel="nofollow">link3</a></p> +`, string(result)) +} diff --git a/modules/markup/markdown/math/block_renderer.go b/modules/markup/markdown/math/block_renderer.go index 412e4d0dee..95a336a02c 100644 --- a/modules/markup/markdown/math/block_renderer.go +++ b/modules/markup/markdown/math/block_renderer.go @@ -42,7 +42,7 @@ func (r *BlockRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { func (r *BlockRenderer) writeLines(w util.BufWriter, source []byte, n gast.Node) { l := n.Lines().Len() - for i := 0; i < l; i++ { + for i := range l { line := n.Lines().At(i) _, _ = w.Write(util.EscapeHTML(line.Value(source))) } @@ -51,8 +51,8 @@ func (r *BlockRenderer) writeLines(w util.BufWriter, source []byte, n gast.Node) func (r *BlockRenderer) renderBlock(w util.BufWriter, source []byte, node gast.Node, entering bool) (gast.WalkStatus, error) { n := node.(*Block) if entering { - code := giteaUtil.Iif(n.Inline, "", `<pre class="code-block is-loading">`) + `<code class="language-math display">` - _ = r.renderInternal.FormatWithSafeAttrs(w, template.HTML(code)) + codeHTML := giteaUtil.Iif[template.HTML](n.Inline, "", `<pre class="code-block is-loading">`) + `<code class="language-math display">` + _, _ = w.WriteString(string(r.renderInternal.ProtectSafeAttrs(codeHTML))) r.writeLines(w, source, n) } else { _, _ = w.WriteString(`</code>` + giteaUtil.Iif(n.Inline, "", `</pre>`) + "\n") diff --git a/modules/markup/markdown/math/inline_parser.go b/modules/markup/markdown/math/inline_parser.go index a57abe9f9b..a711d1e1cd 100644 --- a/modules/markup/markdown/math/inline_parser.go +++ b/modules/markup/markdown/math/inline_parser.go @@ -15,26 +15,26 @@ type inlineParser struct { trigger []byte endBytesSingleDollar []byte endBytesDoubleDollar []byte - endBytesBracket []byte + endBytesParentheses []byte + enableInlineDollar bool } -var defaultInlineDollarParser = &inlineParser{ - trigger: []byte{'$'}, - endBytesSingleDollar: []byte{'$'}, - endBytesDoubleDollar: []byte{'$', '$'}, -} - -func NewInlineDollarParser() parser.InlineParser { - return defaultInlineDollarParser +func NewInlineDollarParser(enableInlineDollar bool) parser.InlineParser { + return &inlineParser{ + trigger: []byte{'$'}, + endBytesSingleDollar: []byte{'$'}, + endBytesDoubleDollar: []byte{'$', '$'}, + enableInlineDollar: enableInlineDollar, + } } -var defaultInlineBracketParser = &inlineParser{ - trigger: []byte{'\\', '('}, - endBytesBracket: []byte{'\\', ')'}, +var defaultInlineParenthesesParser = &inlineParser{ + trigger: []byte{'\\', '('}, + endBytesParentheses: []byte{'\\', ')'}, } -func NewInlineBracketParser() parser.InlineParser { - return defaultInlineBracketParser +func NewInlineParenthesesParser() parser.InlineParser { + return defaultInlineParenthesesParser } // Trigger triggers this parser on $ or \ @@ -46,7 +46,7 @@ func isPunctuation(b byte) bool { return b == '.' || b == '!' || b == '?' || b == ',' || b == ';' || b == ':' } -func isBracket(b byte) bool { +func isParenthesesClose(b byte) bool { return b == ')' } @@ -70,10 +70,11 @@ func (parser *inlineParser) Parse(parent ast.Node, block text.Reader, pc parser. startMarkLen = 1 stopMark = parser.endBytesSingleDollar if len(line) > 1 { - if line[1] == '$' { + switch line[1] { + case '$': startMarkLen = 2 stopMark = parser.endBytesDoubleDollar - } else if line[1] == '`' { + case '`': pos := 1 for ; pos < len(line) && line[pos] == '`'; pos++ { } @@ -85,7 +86,11 @@ func (parser *inlineParser) Parse(parent ast.Node, block text.Reader, pc parser. } } else { startMarkLen = 2 - stopMark = parser.endBytesBracket + stopMark = parser.endBytesParentheses + } + + if line[0] == '$' && !parser.enableInlineDollar && (len(line) == 1 || line[1] != '`') { + return nil } if checkSurrounding { @@ -109,7 +114,7 @@ func (parser *inlineParser) Parse(parent ast.Node, block text.Reader, pc parser. succeedingCharacter = line[i+len(stopMark)] } // check valid ending character - isValidEndingChar := isPunctuation(succeedingCharacter) || isBracket(succeedingCharacter) || + isValidEndingChar := isPunctuation(succeedingCharacter) || isParenthesesClose(succeedingCharacter) || succeedingCharacter == ' ' || succeedingCharacter == '\n' || succeedingCharacter == 0 if checkSurrounding && !isValidEndingChar { break @@ -121,9 +126,10 @@ func (parser *inlineParser) Parse(parent ast.Node, block text.Reader, pc parser. i++ continue } - if line[i] == '{' { + switch line[i] { + case '{': depth++ - } else if line[i] == '}' { + case '}': depth-- } } diff --git a/modules/markup/markdown/math/inline_renderer.go b/modules/markup/markdown/math/inline_renderer.go index d000a7b317..eeeb60cc7e 100644 --- a/modules/markup/markdown/math/inline_renderer.go +++ b/modules/markup/markdown/math/inline_renderer.go @@ -28,7 +28,7 @@ func NewInlineRenderer(renderInternal *internal.RenderInternal) renderer.NodeRen func (r *InlineRenderer) renderInline(w util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) { if entering { - _ = r.renderInternal.FormatWithSafeAttrs(w, `<code class="language-math">`) + _, _ = w.WriteString(string(r.renderInternal.ProtectSafeAttrs(`<code class="language-math">`))) for c := n.FirstChild(); c != nil; c = c.NextSibling() { segment := c.(*ast.Text).Segment value := util.EscapeHTML(segment.Value(source)) diff --git a/modules/markup/markdown/math/math.go b/modules/markup/markdown/math/math.go index a6ff593d62..4b74db2d76 100644 --- a/modules/markup/markdown/math/math.go +++ b/modules/markup/markdown/math/math.go @@ -14,10 +14,11 @@ import ( ) type Options struct { - Enabled bool - ParseDollarInline bool - ParseDollarBlock bool - ParseSquareBlock bool + Enabled bool + ParseInlineDollar bool // inline $$ xxx $$ text + ParseInlineParentheses bool // inline \( xxx \) text + ParseBlockDollar bool // block $$ multiple-line $$ text + ParseBlockSquareBrackets bool // block \[ multiple-line \] text } // Extension is a math extension @@ -42,16 +43,16 @@ func (e *Extension) Extend(m goldmark.Markdown) { return } - inlines := []util.PrioritizedValue{util.Prioritized(NewInlineBracketParser(), 501)} - if e.options.ParseDollarInline { - inlines = append(inlines, util.Prioritized(NewInlineDollarParser(), 502)) + var inlines []util.PrioritizedValue + if e.options.ParseInlineParentheses { + inlines = append(inlines, util.Prioritized(NewInlineParenthesesParser(), 501)) } - m.Parser().AddOptions(parser.WithInlineParsers(inlines...)) + inlines = append(inlines, util.Prioritized(NewInlineDollarParser(e.options.ParseInlineDollar), 502)) + m.Parser().AddOptions(parser.WithInlineParsers(inlines...)) m.Parser().AddOptions(parser.WithBlockParsers( - util.Prioritized(NewBlockParser(e.options.ParseDollarBlock, e.options.ParseSquareBlock), 701), + util.Prioritized(NewBlockParser(e.options.ParseBlockDollar, e.options.ParseBlockSquareBrackets), 701), )) - m.Renderer().AddOptions(renderer.WithNodeRenderers( util.Prioritized(NewBlockRenderer(e.renderInternal), 501), util.Prioritized(NewInlineRenderer(e.renderInternal), 502), diff --git a/modules/markup/markdown/meta_test.go b/modules/markup/markdown/meta_test.go index 278c33f1d2..283d289d48 100644 --- a/modules/markup/markdown/meta_test.go +++ b/modules/markup/markdown/meta_test.go @@ -51,7 +51,7 @@ func TestExtractMetadata(t *testing.T) { var meta IssueTemplate body, err := ExtractMetadata(fmt.Sprintf("%s\n%s\n%s", sepTest, frontTest, sepTest), &meta) assert.NoError(t, err) - assert.Equal(t, "", body) + assert.Empty(t, body) assert.Equal(t, metaTest, meta) assert.True(t, meta.Valid()) }) @@ -60,7 +60,7 @@ func TestExtractMetadata(t *testing.T) { func TestExtractMetadataBytes(t *testing.T) { t.Run("ValidFrontAndBody", func(t *testing.T) { var meta IssueTemplate - body, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s\n%s", sepTest, frontTest, sepTest, bodyTest)), &meta) + body, err := ExtractMetadataBytes(fmt.Appendf(nil, "%s\n%s\n%s\n%s", sepTest, frontTest, sepTest, bodyTest), &meta) assert.NoError(t, err) assert.Equal(t, bodyTest, string(body)) assert.Equal(t, metaTest, meta) @@ -69,21 +69,21 @@ func TestExtractMetadataBytes(t *testing.T) { t.Run("NoFirstSeparator", func(t *testing.T) { var meta IssueTemplate - _, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s", frontTest, sepTest, bodyTest)), &meta) + _, err := ExtractMetadataBytes(fmt.Appendf(nil, "%s\n%s\n%s", frontTest, sepTest, bodyTest), &meta) assert.Error(t, err) }) t.Run("NoLastSeparator", func(t *testing.T) { var meta IssueTemplate - _, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s", sepTest, frontTest, bodyTest)), &meta) + _, err := ExtractMetadataBytes(fmt.Appendf(nil, "%s\n%s\n%s", sepTest, frontTest, bodyTest), &meta) assert.Error(t, err) }) t.Run("NoBody", func(t *testing.T) { var meta IssueTemplate - body, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s", sepTest, frontTest, sepTest)), &meta) + body, err := ExtractMetadataBytes(fmt.Appendf(nil, "%s\n%s\n%s", sepTest, frontTest, sepTest), &meta) assert.NoError(t, err) - assert.Equal(t, "", string(body)) + assert.Empty(t, string(body)) assert.Equal(t, metaTest, meta) assert.True(t, meta.Valid()) }) diff --git a/modules/markup/markdown/renderconfig.go b/modules/markup/markdown/renderconfig.go index f4c48d1b3d..d8b1b10ce6 100644 --- a/modules/markup/markdown/renderconfig.go +++ b/modules/markup/markdown/renderconfig.go @@ -16,7 +16,6 @@ import ( // RenderConfig represents rendering configuration for this file type RenderConfig struct { Meta markup.RenderMetaMode - Icon string TOC string // "false": hide, "side"/empty: in sidebar, "main"/"true": in main view Lang string yamlNode *yaml.Node @@ -74,7 +73,7 @@ func (rc *RenderConfig) UnmarshalYAML(value *yaml.Node) error { type yamlRenderConfig struct { Meta *string `yaml:"meta"` - Icon *string `yaml:"details_icon"` + Icon *string `yaml:"details_icon"` // deprecated, because there is no font icon, so no custom icon TOC *string `yaml:"include_toc"` Lang *string `yaml:"lang"` } @@ -96,10 +95,6 @@ func (rc *RenderConfig) UnmarshalYAML(value *yaml.Node) error { rc.Meta = renderMetaModeFromString(*cfg.Gitea.Meta) } - if cfg.Gitea.Icon != nil { - rc.Icon = strings.TrimSpace(strings.ToLower(*cfg.Gitea.Icon)) - } - if cfg.Gitea.Lang != nil && *cfg.Gitea.Lang != "" { rc.Lang = *cfg.Gitea.Lang } @@ -111,7 +106,7 @@ func (rc *RenderConfig) UnmarshalYAML(value *yaml.Node) error { return nil } -func (rc *RenderConfig) toMetaNode() ast.Node { +func (rc *RenderConfig) toMetaNode(g *ASTTransformer) ast.Node { if rc.yamlNode == nil { return nil } @@ -119,7 +114,7 @@ func (rc *RenderConfig) toMetaNode() ast.Node { case markup.RenderMetaAsTable: return nodeToTable(rc.yamlNode) case markup.RenderMetaAsDetails: - return nodeToDetails(rc.yamlNode, rc.Icon) + return nodeToDetails(g, rc.yamlNode) default: return nil } diff --git a/modules/markup/markdown/renderconfig_test.go b/modules/markup/markdown/renderconfig_test.go index 13346570fa..53c52177a7 100644 --- a/modules/markup/markdown/renderconfig_test.go +++ b/modules/markup/markdown/renderconfig_test.go @@ -21,42 +21,36 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { { "empty", &RenderConfig{ Meta: "table", - Icon: "table", Lang: "", }, "", }, { "lang", &RenderConfig{ Meta: "table", - Icon: "table", Lang: "test", }, "lang: test", }, { "metatable", &RenderConfig{ Meta: "table", - Icon: "table", Lang: "", }, "gitea: table", }, { "metanone", &RenderConfig{ Meta: "none", - Icon: "table", Lang: "", }, "gitea: none", }, { "metadetails", &RenderConfig{ Meta: "details", - Icon: "table", Lang: "", }, "gitea: details", }, { "metawrong", &RenderConfig{ Meta: "details", - Icon: "table", Lang: "", }, "gitea: wrong", }, @@ -64,7 +58,6 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { "toc", &RenderConfig{ TOC: "true", Meta: "table", - Icon: "table", Lang: "", }, "include_toc: true", }, @@ -72,14 +65,12 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { "tocfalse", &RenderConfig{ TOC: "false", Meta: "table", - Icon: "table", Lang: "", }, "include_toc: false", }, { "toclang", &RenderConfig{ Meta: "table", - Icon: "table", TOC: "true", Lang: "testlang", }, ` @@ -90,7 +81,6 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { { "complexlang", &RenderConfig{ Meta: "table", - Icon: "table", Lang: "testlang", }, ` gitea: @@ -100,7 +90,6 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { { "complexlang2", &RenderConfig{ Meta: "table", - Icon: "table", Lang: "testlang", }, ` lang: notright @@ -111,7 +100,6 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { { "complexlang", &RenderConfig{ Meta: "table", - Icon: "table", Lang: "testlang", }, ` gitea: @@ -123,7 +111,6 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { Lang: "two", Meta: "table", TOC: "true", - Icon: "smiley", }, ` lang: one include_toc: true @@ -139,14 +126,12 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { t.Run(tt.name, func(t *testing.T) { got := &RenderConfig{ Meta: "table", - Icon: "table", Lang: "", } err := yaml.Unmarshal([]byte(strings.ReplaceAll(tt.args, "\t", " ")), got) require.NoError(t, err) assert.Equal(t, tt.expected.Meta, got.Meta) - assert.Equal(t, tt.expected.Icon, got.Icon) assert.Equal(t, tt.expected.Lang, got.Lang) assert.Equal(t, tt.expected.TOC, got.TOC) }) diff --git a/modules/markup/markdown/toc.go b/modules/markup/markdown/toc.go index ea1af83a3e..a11b9d0390 100644 --- a/modules/markup/markdown/toc.go +++ b/modules/markup/markdown/toc.go @@ -4,7 +4,6 @@ package markdown import ( - "fmt" "net/url" "code.gitea.io/gitea/modules/translation" @@ -50,7 +49,7 @@ func createTOCNode(toc []Header, lang string, detailsAttrs map[string]string) as } li := ast.NewListItem(currentLevel * 2) a := ast.NewLink() - a.Destination = []byte(fmt.Sprintf("#%s", url.QueryEscape(header.ID))) + a.Destination = []byte("#" + url.QueryEscape(header.ID)) a.AppendChild(a, ast.NewString([]byte(header.Text))) li.AppendChild(li, a) ul.AppendChild(ul, li) diff --git a/modules/markup/markdown/transform_blockquote.go b/modules/markup/markdown/transform_blockquote.go index 2651d44a69..bf17f01681 100644 --- a/modules/markup/markdown/transform_blockquote.go +++ b/modules/markup/markdown/transform_blockquote.go @@ -46,7 +46,7 @@ func (g *ASTTransformer) extractBlockquoteAttentionEmphasis(firstParagraph ast.N if !ok { return "", nil } - val1 := string(node1.Text(reader.Source())) //nolint:staticcheck + val1 := string(node1.Text(reader.Source())) //nolint:staticcheck // Text is deprecated attentionType := strings.ToLower(val1) if g.attentionTypes.Contains(attentionType) { return attentionType, []ast.Node{node1} @@ -115,6 +115,9 @@ func (g *ASTTransformer) transformBlockquote(v *ast.Blockquote, reader text.Read // grab these nodes and make sure we adhere to the attention blockquote structure firstParagraph := v.FirstChild() + if firstParagraph == nil { + return ast.WalkContinue, nil + } g.applyElementDir(firstParagraph) attentionType, processedNodes := g.extractBlockquoteAttentionEmphasis(firstParagraph, reader) diff --git a/modules/markup/markdown/transform_codespan.go b/modules/markup/markdown/transform_codespan.go index bccc43aad2..c2e4295bc2 100644 --- a/modules/markup/markdown/transform_codespan.go +++ b/modules/markup/markdown/transform_codespan.go @@ -68,7 +68,7 @@ func cssColorHandler(value string) bool { } func (g *ASTTransformer) transformCodeSpan(_ *markup.RenderContext, v *ast.CodeSpan, reader text.Reader) { - colorContent := v.Text(reader.Source()) //nolint:staticcheck + colorContent := v.Text(reader.Source()) //nolint:staticcheck // Text is deprecated if cssColorHandler(string(colorContent)) { v.AppendChild(v, NewColorPreview(colorContent)) } diff --git a/modules/markup/markdown/transform_heading.go b/modules/markup/markdown/transform_heading.go index 5f8a12794d..a229a7b1a4 100644 --- a/modules/markup/markdown/transform_heading.go +++ b/modules/markup/markdown/transform_heading.go @@ -16,10 +16,10 @@ import ( func (g *ASTTransformer) transformHeading(_ *markup.RenderContext, v *ast.Heading, reader text.Reader, tocList *[]Header) { for _, attr := range v.Attributes() { if _, ok := attr.Value.([]byte); !ok { - v.SetAttribute(attr.Name, []byte(fmt.Sprintf("%v", attr.Value))) + v.SetAttribute(attr.Name, fmt.Appendf(nil, "%v", attr.Value)) } } - txt := v.Text(reader.Source()) //nolint:staticcheck + txt := v.Text(reader.Source()) //nolint:staticcheck // Text is deprecated header := Header{ Text: util.UnsafeBytesToString(txt), Level: v.Level, diff --git a/modules/markup/markdown/transform_image.go b/modules/markup/markdown/transform_image.go deleted file mode 100644 index 36512e59a8..0000000000 --- a/modules/markup/markdown/transform_image.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package markdown - -import ( - "code.gitea.io/gitea/modules/markup" - - "github.com/yuin/goldmark/ast" -) - -func (g *ASTTransformer) transformImage(ctx *markup.RenderContext, v *ast.Image) { - // Images need two things: - // - // 1. Their src needs to munged to be a real value - // 2. If they're not wrapped with a link they need a link wrapper - - // Check if the destination is a real link - if len(v.Destination) > 0 && !markup.IsFullURLBytes(v.Destination) { - v.Destination = []byte(ctx.RenderHelper.ResolveLink(string(v.Destination), markup.LinkTypeMedia)) - } - - parent := v.Parent() - // Create a link around image only if parent is not already a link - if _, ok := parent.(*ast.Link); !ok && parent != nil { - next := v.NextSibling() - - // Create a link wrapper - wrap := ast.NewLink() - wrap.Destination = v.Destination - wrap.Title = v.Title - wrap.SetAttributeString("target", []byte("_blank")) - - // Duplicate the current image node - image := ast.NewImage(ast.NewLink()) - image.Destination = v.Destination - image.Title = v.Title - for _, attr := range v.Attributes() { - image.SetAttribute(attr.Name, attr.Value) - } - for child := v.FirstChild(); child != nil; { - next := child.NextSibling() - image.AppendChild(image, child) - child = next - } - - // Append our duplicate image to the wrapper link - wrap.AppendChild(wrap, image) - - // Wire in the next sibling - wrap.SetNextSibling(next) - - // Replace the current node with the wrapper link - parent.ReplaceChild(parent, v, wrap) - - // But most importantly ensure the next sibling is still on the old image too - v.SetNextSibling(next) - } -} diff --git a/modules/markup/markdown/transform_link.go b/modules/markup/markdown/transform_link.go deleted file mode 100644 index 51c2c915d8..0000000000 --- a/modules/markup/markdown/transform_link.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package markdown - -import ( - "code.gitea.io/gitea/modules/markup" - - "github.com/yuin/goldmark/ast" -) - -func resolveLink(ctx *markup.RenderContext, link, userContentAnchorPrefix string) (result string, resolved bool) { - isAnchorFragment := link != "" && link[0] == '#' - if !isAnchorFragment && !markup.IsFullURLString(link) { - link, resolved = ctx.RenderHelper.ResolveLink(link, markup.LinkTypeDefault), true - } - if isAnchorFragment && userContentAnchorPrefix != "" { - link, resolved = userContentAnchorPrefix+link[1:], true - } - return link, resolved -} - -func (g *ASTTransformer) transformLink(ctx *markup.RenderContext, v *ast.Link) { - if link, resolved := resolveLink(ctx, string(v.Destination), "#user-content-"); resolved { - v.Destination = []byte(link) - } -} diff --git a/modules/markup/mdstripper/mdstripper.go b/modules/markup/mdstripper/mdstripper.go index fe0eabb473..6e392444b4 100644 --- a/modules/markup/mdstripper/mdstripper.go +++ b/modules/markup/mdstripper/mdstripper.go @@ -46,7 +46,7 @@ func (r *stripRenderer) Render(w io.Writer, source []byte, doc ast.Node) error { coalesce := prevSibIsText r.processString( w, - v.Text(source), //nolint:staticcheck + v.Text(source), //nolint:staticcheck // Text is deprecated coalesce) if v.SoftLineBreak() { r.doubleSpace(w) @@ -107,11 +107,12 @@ func (r *stripRenderer) processAutoLink(w io.Writer, link []byte) { } var sep string - if parts[3] == "issues" { + switch parts[3] { + case "issues": sep = "#" - } else if parts[3] == "pulls" { + case "pulls": sep = "!" - } else { + default: // Process out of band r.links = append(r.links, linkStr) return diff --git a/modules/markup/mdstripper/mdstripper_test.go b/modules/markup/mdstripper/mdstripper_test.go index ea34df0a3b..7fb49c1e01 100644 --- a/modules/markup/mdstripper/mdstripper_test.go +++ b/modules/markup/mdstripper/mdstripper_test.go @@ -79,7 +79,7 @@ A HIDDEN ` + "`" + `GHOST` + "`" + ` IN THIS LINE. lines = append(lines, line) } } - assert.EqualValues(t, test.expectedText, lines) - assert.EqualValues(t, test.expectedLinks, links) + assert.Equal(t, test.expectedText, lines) + assert.Equal(t, test.expectedLinks, links) } } diff --git a/modules/markup/orgmode/orgmode.go b/modules/markup/orgmode/orgmode.go index 70d02c1321..93c335d244 100644 --- a/modules/markup/orgmode/orgmode.go +++ b/modules/markup/orgmode/orgmode.go @@ -1,7 +1,7 @@ // Copyright 2017 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package markup +package orgmode import ( "fmt" @@ -125,27 +125,13 @@ type orgWriter struct { var _ org.Writer = (*orgWriter)(nil) -func (r *orgWriter) resolveLink(kind, link string) string { - link = strings.TrimPrefix(link, "file:") - if !strings.HasPrefix(link, "#") && // not a URL fragment - !markup.IsFullURLString(link) { - if kind == "regular" { - // orgmode reports the link kind as "regular" for "[[ImageLink.svg][The Image Desc]]" - // so we need to try to guess the link kind again here - kind = org.RegularLink{URL: link}.Kind() - } - if kind == "image" || kind == "video" { - link = r.rctx.RenderHelper.ResolveLink(link, markup.LinkTypeMedia) - } else { - link = r.rctx.RenderHelper.ResolveLink(link, markup.LinkTypeDefault) - } - } - return link +func (r *orgWriter) resolveLink(link string) string { + return strings.TrimPrefix(link, "file:") } // WriteRegularLink renders images, links or videos func (r *orgWriter) WriteRegularLink(l org.RegularLink) { - link := r.resolveLink(l.Kind(), l.URL) + link := r.resolveLink(l.URL) printHTML := func(html template.HTML, a ...any) { _, _ = fmt.Fprint(r, htmlutil.HTMLFormat(html, a...)) @@ -156,14 +142,14 @@ func (r *orgWriter) WriteRegularLink(l org.RegularLink) { if l.Description == nil { printHTML(`<img src="%s" alt="%s">`, link, link) } else { - imageSrc := r.resolveLink(l.Kind(), org.String(l.Description...)) + imageSrc := r.resolveLink(org.String(l.Description...)) printHTML(`<a href="%s"><img src="%s" alt="%s"></a>`, link, imageSrc, imageSrc) } case "video": if l.Description == nil { printHTML(`<video src="%s">%s</video>`, link, link) } else { - videoSrc := r.resolveLink(l.Kind(), org.String(l.Description...)) + videoSrc := r.resolveLink(org.String(l.Description...)) printHTML(`<a href="%s"><video src="%s">%s</video></a>`, link, videoSrc, videoSrc) } default: diff --git a/modules/markup/orgmode/orgmode_test.go b/modules/markup/orgmode/orgmode_test.go index de39bafebe..df4bb38ad1 100644 --- a/modules/markup/orgmode/orgmode_test.go +++ b/modules/markup/orgmode/orgmode_test.go @@ -1,7 +1,7 @@ // Copyright 2017 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package markup +package orgmode_test import ( "os" @@ -9,6 +9,7 @@ import ( "testing" "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/markup/orgmode" "code.gitea.io/gitea/modules/setting" "github.com/stretchr/testify/assert" @@ -22,7 +23,7 @@ func TestMain(m *testing.M) { func TestRender_StandardLinks(t *testing.T) { test := func(input, expected string) { - buffer, err := RenderString(markup.NewTestRenderContext("/relative-path/media/branch/main/"), input) + buffer, err := orgmode.RenderString(markup.NewTestRenderContext(), input) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } @@ -30,37 +31,37 @@ func TestRender_StandardLinks(t *testing.T) { test("[[https://google.com/]]", `<p><a href="https://google.com/">https://google.com/</a></p>`) test("[[ImageLink.svg][The Image Desc]]", - `<p><a href="/relative-path/media/branch/main/ImageLink.svg">The Image Desc</a></p>`) + `<p><a href="ImageLink.svg">The Image Desc</a></p>`) } func TestRender_InternalLinks(t *testing.T) { test := func(input, expected string) { - buffer, err := RenderString(markup.NewTestRenderContext("/relative-path/src/branch/main"), input) + buffer, err := orgmode.RenderString(markup.NewTestRenderContext(), input) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } test("[[file:test.org][Test]]", - `<p><a href="/relative-path/src/branch/main/test.org">Test</a></p>`) + `<p><a href="test.org">Test</a></p>`) test("[[./test.org][Test]]", - `<p><a href="/relative-path/src/branch/main/test.org">Test</a></p>`) + `<p><a href="./test.org">Test</a></p>`) test("[[test.org][Test]]", - `<p><a href="/relative-path/src/branch/main/test.org">Test</a></p>`) + `<p><a href="test.org">Test</a></p>`) test("[[path/to/test.org][Test]]", - `<p><a href="/relative-path/src/branch/main/path/to/test.org">Test</a></p>`) + `<p><a href="path/to/test.org">Test</a></p>`) } func TestRender_Media(t *testing.T) { test := func(input, expected string) { - buffer, err := RenderString(markup.NewTestRenderContext("./relative-path"), input) + buffer, err := orgmode.RenderString(markup.NewTestRenderContext(), input) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } test("[[file:../../.images/src/02/train.jpg]]", - `<p><img src=".images/src/02/train.jpg" alt=".images/src/02/train.jpg"></p>`) + `<p><img src="../../.images/src/02/train.jpg" alt="../../.images/src/02/train.jpg"></p>`) test("[[file:train.jpg]]", - `<p><img src="relative-path/train.jpg" alt="relative-path/train.jpg"></p>`) + `<p><img src="train.jpg" alt="train.jpg"></p>`) // With description. test("[[https://example.com][https://example.com/example.svg]]", @@ -91,7 +92,7 @@ func TestRender_Media(t *testing.T) { func TestRender_Source(t *testing.T) { test := func(input, expected string) { - buffer, err := RenderString(markup.NewTestRenderContext(), input) + buffer, err := orgmode.RenderString(markup.NewTestRenderContext(), input) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } diff --git a/modules/markup/render.go b/modules/markup/render.go index 37a2a86687..79f1f473c2 100644 --- a/modules/markup/render.go +++ b/modules/markup/render.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "net/url" + "strconv" "strings" "time" @@ -46,7 +47,7 @@ type RenderOptions struct { // user&repo, format&style®exp (for external issue pattern), teams&org (for mention) // RefTypeNameSubURL (for iframe&asciicast) // markupAllowShortIssuePattern - // markdownLineBreakStyle (comment, document) + // markdownNewLineHardBreak Metas map[string]string // used by external render. the router "/org/repo/render/..." will output the rendered content in a standalone page @@ -247,7 +248,8 @@ func Init(renderHelpFuncs *RenderHelperFuncs) { } func ComposeSimpleDocumentMetas() map[string]string { - return map[string]string{"markdownLineBreakStyle": "document"} + // TODO: there is no separate config option for "simple document" rendering, so temporarily use the same config as "repo file" + return map[string]string{"markdownNewLineHardBreak": strconv.FormatBool(setting.Markdown.RenderOptionsRepoFile.NewLineHardBreak)} } type TestRenderHelper struct { @@ -261,8 +263,14 @@ func (r *TestRenderHelper) IsCommitIDExisting(commitID string) bool { return strings.HasPrefix(commitID, "65f1bf2") //|| strings.HasPrefix(commitID, "88fc37a") } -func (r *TestRenderHelper) ResolveLink(link string, likeType LinkType) string { - return r.ctx.ResolveLinkRelative(r.BaseLink, "", link) +func (r *TestRenderHelper) ResolveLink(link, preferLinkType string) string { + linkType, link := ParseRenderedLink(link, preferLinkType) + switch linkType { + case LinkTypeRoot: + return r.ctx.ResolveLinkRoot(link) + default: + return r.ctx.ResolveLinkRelative(r.BaseLink, "", link) + } } var _ RenderHelper = (*TestRenderHelper)(nil) diff --git a/modules/markup/render_helper.go b/modules/markup/render_helper.go index 8ff0e7d6fb..b16f1189c5 100644 --- a/modules/markup/render_helper.go +++ b/modules/markup/render_helper.go @@ -10,13 +10,11 @@ import ( "code.gitea.io/gitea/modules/setting" ) -type LinkType string - const ( - LinkTypeApp LinkType = "app" // the link is relative to the AppSubURL - LinkTypeDefault LinkType = "default" // the link is relative to the default base (eg: repo link, or current ref tree path) - LinkTypeMedia LinkType = "media" // the link should be used to access media files (images, videos) - LinkTypeRaw LinkType = "raw" // not really useful, mainly for environment GITEA_PREFIX_RAW for external renders + LinkTypeDefault = "" + LinkTypeRoot = "/:root" // the link is relative to the AppSubURL(ROOT_URL) + LinkTypeMedia = "/:media" // the link should be used to access media files (images, videos) + LinkTypeRaw = "/:raw" // not really useful, mainly for environment GITEA_PREFIX_RAW for external renders ) type RenderHelper interface { @@ -27,7 +25,7 @@ type RenderHelper interface { // but not make processors to guess "is it rendering a comment or a wiki?" or "does it need to check commit ID?" IsCommitIDExisting(commitID string) bool - ResolveLink(link string, likeType LinkType) string + ResolveLink(link, preferLinkType string) string } // RenderHelperFuncs is used to decouple cycle-import @@ -51,7 +49,8 @@ func (r *SimpleRenderHelper) IsCommitIDExisting(commitID string) bool { return false } -func (r *SimpleRenderHelper) ResolveLink(link string, likeType LinkType) string { +func (r *SimpleRenderHelper) ResolveLink(link, preferLinkType string) string { + _, link = ParseRenderedLink(link, preferLinkType) return resolveLinkRelative(context.Background(), setting.AppSubURL+"/", "", link, false) } diff --git a/modules/markup/render_link.go b/modules/markup/render_link.go index b2e0699681..046544ce81 100644 --- a/modules/markup/render_link.go +++ b/modules/markup/render_link.go @@ -33,10 +33,24 @@ func resolveLinkRelative(ctx context.Context, base, cur, link string, absolute b return finalLink } -func (ctx *RenderContext) ResolveLinkRelative(base, cur, link string) (finalLink string) { +func (ctx *RenderContext) ResolveLinkRelative(base, cur, link string) string { + if strings.HasPrefix(link, "/:") { + setting.PanicInDevOrTesting("invalid link %q, forgot to cut?", link) + } return resolveLinkRelative(ctx, base, cur, link, ctx.RenderOptions.UseAbsoluteLink) } -func (ctx *RenderContext) ResolveLinkApp(link string) string { +func (ctx *RenderContext) ResolveLinkRoot(link string) string { return ctx.ResolveLinkRelative(setting.AppSubURL+"/", "", link) } + +func ParseRenderedLink(s, preferLinkType string) (linkType, link string) { + if strings.HasPrefix(s, "/:") { + p := strings.IndexByte(s[1:], '/') + if p == -1 { + return s, "" + } + return s[:p+1], s[p+2:] + } + return preferLinkType, s +} diff --git a/modules/markup/render_link_test.go b/modules/markup/render_link_test.go index c904ec7f18..972e15308c 100644 --- a/modules/markup/render_link_test.go +++ b/modules/markup/render_link_test.go @@ -4,7 +4,6 @@ package markup import ( - "context" "testing" "code.gitea.io/gitea/modules/setting" @@ -13,7 +12,7 @@ import ( ) func TestResolveLinkRelative(t *testing.T) { - ctx := context.Background() + ctx := t.Context() setting.AppURL = "http://localhost:3000" assert.Equal(t, "/a", resolveLinkRelative(ctx, "/a", "", "", false)) assert.Equal(t, "/a/b", resolveLinkRelative(ctx, "/a", "b", "", false)) diff --git a/modules/markup/renderer.go b/modules/markup/renderer.go index 35f90eb46c..b6e9c348b7 100644 --- a/modules/markup/renderer.go +++ b/modules/markup/renderer.go @@ -4,12 +4,12 @@ package markup import ( - "bytes" "io" "path" "strings" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/typesniffer" ) // Renderer defines an interface for rendering markup file to HTML @@ -37,7 +37,7 @@ type ExternalRenderer interface { // RendererContentDetector detects if the content can be rendered // by specified renderer type RendererContentDetector interface { - CanRender(filename string, input io.Reader) bool + CanRender(filename string, sniffedType typesniffer.SniffedType, prefetchBuf []byte) bool } var ( @@ -60,13 +60,9 @@ func GetRendererByFileName(filename string) Renderer { } // DetectRendererType detects the markup type of the content -func DetectRendererType(filename string, input io.Reader) string { - buf, err := io.ReadAll(input) - if err != nil { - return "" - } +func DetectRendererType(filename string, sniffedType typesniffer.SniffedType, prefetchBuf []byte) string { for _, renderer := range renderers { - if detector, ok := renderer.(RendererContentDetector); ok && detector.CanRender(filename, bytes.NewReader(buf)) { + if detector, ok := renderer.(RendererContentDetector); ok && detector.CanRender(filename, sniffedType, prefetchBuf) { return renderer.Name() } } diff --git a/modules/markup/renderer_test.go b/modules/markup/renderer_test.go deleted file mode 100644 index 0791081f94..0000000000 --- a/modules/markup/renderer_test.go +++ /dev/null @@ -1,4 +0,0 @@ -// Copyright 2017 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package markup_test diff --git a/modules/markup/sanitizer_default.go b/modules/markup/sanitizer_default.go index 14161eb533..0fbf0f0b24 100644 --- a/modules/markup/sanitizer_default.go +++ b/modules/markup/sanitizer_default.go @@ -4,6 +4,7 @@ package markup import ( + "html/template" "io" "net/url" "regexp" @@ -52,6 +53,8 @@ func (st *Sanitizer) createDefaultPolicy() *bluemonday.Policy { policy.AllowAttrs("src", "autoplay", "controls").OnElements("video") + policy.AllowAttrs("loading").OnElements("img") + // Allow generally safe attributes (reference: https://github.com/jch/html-pipeline) generalSafeAttrs := []string{ "abbr", "accept", "accept-charset", @@ -90,9 +93,9 @@ func (st *Sanitizer) createDefaultPolicy() *bluemonday.Policy { return policy } -// Sanitize takes a string that contains a HTML fragment or document and applies policy whitelist. -func Sanitize(s string) string { - return GetDefaultSanitizer().defaultPolicy.Sanitize(s) +// Sanitize use default sanitizer policy to sanitize a string +func Sanitize(s string) template.HTML { + return template.HTML(GetDefaultSanitizer().defaultPolicy.Sanitize(s)) } // SanitizeReader sanitizes a Reader diff --git a/modules/markup/sanitizer_default_test.go b/modules/markup/sanitizer_default_test.go index e6fbae5056..e5ba018e1b 100644 --- a/modules/markup/sanitizer_default_test.go +++ b/modules/markup/sanitizer_default_test.go @@ -62,9 +62,13 @@ func TestSanitizer(t *testing.T) { `<a href="javascript:alert('xss')">bad</a>`, `bad`, `<a href="vbscript:no">bad</a>`, `bad`, `<a href="data:1234">bad</a>`, `bad`, + + // Some classes and attributes are used by the frontend framework and will execute JS code, so make sure they are removed + `<div class="link-action" data-attr-class="foo" data-url="xxx">txt</div>`, `<div data-attr-class="foo">txt</div>`, + `<div class="form-fetch-action" data-markdown-generated-content="bar" data-global-init="a" data-global-click="b">txt</div>`, `<div data-markdown-generated-content="bar">txt</div>`, } for i := 0; i < len(testCases); i += 2 { - assert.Equal(t, testCases[i+1], Sanitize(testCases[i])) + assert.Equal(t, testCases[i+1], string(Sanitize(testCases[i]))) } } |