diff options
author | Lunny Xiao <xiaolunwen@gmail.com> | 2021-04-20 06:25:08 +0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-04-19 18:25:08 -0400 |
commit | 9d99f6ab19ac3f97af3ca126720e9075c127a652 (patch) | |
tree | b817b4582a871f83b91ad7977fe772fc3501c1e8 | |
parent | c9cc6698d2172625854cd063301e63602204a2a1 (diff) | |
download | gitea-9d99f6ab19ac3f97af3ca126720e9075c127a652.tar.gz gitea-9d99f6ab19ac3f97af3ca126720e9075c127a652.zip |
Refactor renders (#15175)
* Refactor renders
* Some performance optimization
* Fix comment
* Transform reader
* Fix csv test
* Fix test
* Fix tests
* Improve optimaziation
* Fix test
* Fix test
* Detect file encoding with reader
* Improve optimaziation
* reduce memory usage
* improve code
* fix build
* Fix test
* Fix for go1.15
* Fix render
* Fix comment
* Fix lint
* Fix test
* Don't use NormalEOF when unnecessary
* revert change on util.go
* Apply suggestions from code review
Co-authored-by: zeripath <art27@cantab.net>
* rename function
* Take NormalEOF back
Co-authored-by: zeripath <art27@cantab.net>
41 files changed, 1026 insertions, 626 deletions
diff --git a/contrib/pr/checkout.go b/contrib/pr/checkout.go index 63eca484a5..9ee692fd35 100644 --- a/contrib/pr/checkout.go +++ b/contrib/pr/checkout.go @@ -114,7 +114,7 @@ func runPR() { log.Printf("[PR] Setting up router\n") //routers.GlobalInit() - external.RegisterParsers() + external.RegisterRenderers() markup.Init() c := routes.NormalRoutes() diff --git a/models/issue_comment.go b/models/issue_comment.go index 53d4d638c4..26bf122dc9 100644 --- a/models/issue_comment.go +++ b/models/issue_comment.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/references" "code.gitea.io/gitea/modules/structs" @@ -1178,8 +1179,13 @@ func findCodeComments(e Engine, opts FindCommentsOptions, issue *Issue, currentU return nil, err } - comment.RenderedContent = string(markdown.Render([]byte(comment.Content), issue.Repo.Link(), - issue.Repo.ComposeMetas())) + var err error + if comment.RenderedContent, err = markdown.RenderString(&markup.RenderContext{ + URLPrefix: issue.Repo.Link(), + Metas: issue.Repo.ComposeMetas(), + }, comment.Content); err != nil { + return nil, err + } } return comments[:n], nil } diff --git a/models/repo.go b/models/repo.go index bdb84ee00d..fc673cace8 100644 --- a/models/repo.go +++ b/models/repo.go @@ -863,7 +863,10 @@ func (repo *Repository) getUsersWithAccessMode(e Engine, mode AccessMode) (_ []* // DescriptionHTML does special handles to description and return HTML string. func (repo *Repository) DescriptionHTML() template.HTML { - desc, err := markup.RenderDescriptionHTML([]byte(repo.Description), repo.HTMLURL(), repo.ComposeMetas()) + desc, err := markup.RenderDescriptionHTML(&markup.RenderContext{ + URLPrefix: repo.HTMLURL(), + Metas: repo.ComposeMetas(), + }, repo.Description) if err != nil { log.Error("Failed to render description for %s (ID: %d): %v", repo.Name, repo.ID, err) return template.HTML(markup.Sanitize(repo.Description)) diff --git a/models/repo_generate.go b/models/repo_generate.go index b0016494c4..1cf73bc55e 100644 --- a/models/repo_generate.go +++ b/models/repo_generate.go @@ -5,13 +5,14 @@ package models import ( + "bufio" + "bytes" "strconv" "strings" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/storage" - "code.gitea.io/gitea/modules/util" "github.com/gobwas/glob" ) @@ -49,9 +50,9 @@ func (gt GiteaTemplate) Globs() []glob.Glob { } gt.globs = make([]glob.Glob, 0) - lines := strings.Split(string(util.NormalizeEOL(gt.Content)), "\n") - for _, line := range lines { - line = strings.TrimSpace(line) + scanner := bufio.NewScanner(bytes.NewReader(gt.Content)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) if line == "" || strings.HasPrefix(line, "#") { continue } diff --git a/modules/charset/charset.go b/modules/charset/charset.go index a7e427db99..3000864c2e 100644 --- a/modules/charset/charset.go +++ b/modules/charset/charset.go @@ -7,6 +7,8 @@ package charset import ( "bytes" "fmt" + "io" + "io/ioutil" "strings" "unicode/utf8" @@ -21,6 +23,33 @@ import ( // UTF8BOM is the utf-8 byte-order marker var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'} +// ToUTF8WithFallbackReader detects the encoding of content and coverts to UTF-8 reader if possible +func ToUTF8WithFallbackReader(rd io.Reader) io.Reader { + var buf = make([]byte, 2048) + n, err := rd.Read(buf) + if err != nil { + return rd + } + + charsetLabel, err := DetectEncoding(buf[:n]) + if err != nil || charsetLabel == "UTF-8" { + return io.MultiReader(bytes.NewReader(RemoveBOMIfPresent(buf[:n])), rd) + } + + encoding, _ := charset.Lookup(charsetLabel) + if encoding == nil { + return io.MultiReader(bytes.NewReader(buf[:n]), rd) + } + + return transform.NewReader( + io.MultiReader( + bytes.NewReader(RemoveBOMIfPresent(buf[:n])), + rd, + ), + encoding.NewDecoder(), + ) +} + // ToUTF8WithErr converts content to UTF8 encoding func ToUTF8WithErr(content []byte) (string, error) { charsetLabel, err := DetectEncoding(content) @@ -49,24 +78,8 @@ func ToUTF8WithErr(content []byte) (string, error) { // ToUTF8WithFallback detects the encoding of content and coverts to UTF-8 if possible func ToUTF8WithFallback(content []byte) []byte { - charsetLabel, err := DetectEncoding(content) - if err != nil || charsetLabel == "UTF-8" { - return RemoveBOMIfPresent(content) - } - - encoding, _ := charset.Lookup(charsetLabel) - if encoding == nil { - return content - } - - // If there is an error, we concatenate the nicely decoded part and the - // original left over. This way we won't lose data. - result, n, err := transform.Bytes(encoding.NewDecoder(), content) - if err != nil { - return append(result, content[n:]...) - } - - return RemoveBOMIfPresent(result) + bs, _ := ioutil.ReadAll(ToUTF8WithFallbackReader(bytes.NewReader(content))) + return bs } // ToUTF8 converts content to UTF8 encoding and ignore error diff --git a/modules/csv/csv.go b/modules/csv/csv.go index 1aa78fdeec..bf433f77d2 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -7,7 +7,9 @@ package csv import ( "bytes" "encoding/csv" + stdcsv "encoding/csv" "errors" + "io" "regexp" "strings" @@ -18,17 +20,31 @@ import ( var quoteRegexp = regexp.MustCompile(`["'][\s\S]+?["']`) // CreateReader creates a csv.Reader with the given delimiter. -func CreateReader(rawBytes []byte, delimiter rune) *csv.Reader { - rd := csv.NewReader(bytes.NewReader(rawBytes)) +func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader { + rd := stdcsv.NewReader(input) rd.Comma = delimiter rd.TrimLeadingSpace = true return rd } // CreateReaderAndGuessDelimiter tries to guess the field delimiter from the content and creates a csv.Reader. -func CreateReaderAndGuessDelimiter(rawBytes []byte) *csv.Reader { - delimiter := guessDelimiter(rawBytes) - return CreateReader(rawBytes, delimiter) +func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) { + var data = make([]byte, 1e4) + size, err := rd.Read(data) + if err != nil { + return nil, err + } + + delimiter := guessDelimiter(data[:size]) + + var newInput io.Reader + if size < 1e4 { + newInput = bytes.NewReader(data[:size]) + } else { + newInput = io.MultiReader(bytes.NewReader(data), rd) + } + + return CreateReader(newInput, delimiter), nil } // guessDelimiter scores the input CSV data against delimiters, and returns the best match. diff --git a/modules/csv/csv_test.go b/modules/csv/csv_test.go index 3a7584e21d..3cc09c40aa 100644 --- a/modules/csv/csv_test.go +++ b/modules/csv/csv_test.go @@ -5,20 +5,23 @@ package csv import ( + "bytes" + "strings" "testing" "github.com/stretchr/testify/assert" ) func TestCreateReader(t *testing.T) { - rd := CreateReader([]byte{}, ',') + rd := CreateReader(bytes.NewReader([]byte{}), ',') assert.Equal(t, ',', rd.Comma) } func TestCreateReaderAndGuessDelimiter(t *testing.T) { input := "a;b;c\n1;2;3\n4;5;6" - rd := CreateReaderAndGuessDelimiter([]byte(input)) + rd, err := CreateReaderAndGuessDelimiter(strings.NewReader(input)) + assert.NoError(t, err) assert.Equal(t, ';', rd.Comma) } diff --git a/modules/markup/csv/csv.go b/modules/markup/csv/csv.go index 68c89166b5..6572b0ee1e 100644 --- a/modules/markup/csv/csv.go +++ b/modules/markup/csv/csv.go @@ -5,9 +5,11 @@ package markup import ( + "bufio" "bytes" "html" "io" + "io/ioutil" "strconv" "code.gitea.io/gitea/modules/csv" @@ -16,55 +18,89 @@ import ( ) func init() { - markup.RegisterParser(Parser{}) + markup.RegisterRenderer(Renderer{}) } -// Parser implements markup.Parser for csv files -type Parser struct { +// Renderer implements markup.Renderer for csv files +type Renderer struct { } -// Name implements markup.Parser -func (Parser) Name() string { +// Name implements markup.Renderer +func (Renderer) Name() string { return "csv" } -// NeedPostProcess implements markup.Parser -func (Parser) NeedPostProcess() bool { return false } +// NeedPostProcess implements markup.Renderer +func (Renderer) NeedPostProcess() bool { return false } -// Extensions implements markup.Parser -func (Parser) Extensions() []string { +// Extensions implements markup.Renderer +func (Renderer) Extensions() []string { return []string{".csv", ".tsv"} } -// Render implements markup.Parser -func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte { - var tmpBlock bytes.Buffer - - if setting.UI.CSV.MaxFileSize != 0 && setting.UI.CSV.MaxFileSize < int64(len(rawBytes)) { - tmpBlock.WriteString("<pre>") - tmpBlock.WriteString(html.EscapeString(string(rawBytes))) - tmpBlock.WriteString("</pre>") - return tmpBlock.Bytes() +func writeField(w io.Writer, element, class, field string) error { + if _, err := io.WriteString(w, "<"); err != nil { + return err + } + if _, err := io.WriteString(w, element); err != nil { + return err + } + if len(class) > 0 { + if _, err := io.WriteString(w, " class=\""); err != nil { + return err + } + if _, err := io.WriteString(w, class); err != nil { + return err + } + if _, err := io.WriteString(w, "\""); err != nil { + return err + } + } + if _, err := io.WriteString(w, ">"); err != nil { + return err + } + if _, err := io.WriteString(w, html.EscapeString(field)); err != nil { + return err } + if _, err := io.WriteString(w, "</"); err != nil { + return err + } + if _, err := io.WriteString(w, element); err != nil { + return err + } + _, err := io.WriteString(w, ">") + return err +} - rd := csv.CreateReaderAndGuessDelimiter(rawBytes) +// Render implements markup.Renderer +func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { + var tmpBlock = bufio.NewWriter(output) - writeField := func(element, class, field string) { - tmpBlock.WriteString("<") - tmpBlock.WriteString(element) - if len(class) > 0 { - tmpBlock.WriteString(" class=\"") - tmpBlock.WriteString(class) - tmpBlock.WriteString("\"") + // FIXME: don't read all to memory + rawBytes, err := ioutil.ReadAll(input) + if err != nil { + return err + } + + if setting.UI.CSV.MaxFileSize != 0 && setting.UI.CSV.MaxFileSize < int64(len(rawBytes)) { + if _, err := tmpBlock.WriteString("<pre>"); err != nil { + return err } - tmpBlock.WriteString(">") - tmpBlock.WriteString(html.EscapeString(field)) - tmpBlock.WriteString("</") - tmpBlock.WriteString(element) - tmpBlock.WriteString(">") + if _, err := tmpBlock.WriteString(html.EscapeString(string(rawBytes))); err != nil { + return err + } + _, err = tmpBlock.WriteString("</pre>") + return err + } + + rd, err := csv.CreateReaderAndGuessDelimiter(bytes.NewReader(rawBytes)) + if err != nil { + return err } - tmpBlock.WriteString(`<table class="data-table">`) + if _, err := tmpBlock.WriteString(`<table class="data-table">`); err != nil { + return err + } row := 1 for { fields, err := rd.Read() @@ -74,20 +110,29 @@ func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, if err != nil { continue } - tmpBlock.WriteString("<tr>") + if _, err := tmpBlock.WriteString("<tr>"); err != nil { + return err + } element := "td" if row == 1 { element = "th" } - writeField(element, "line-num", strconv.Itoa(row)) + if err := writeField(tmpBlock, element, "line-num", strconv.Itoa(row)); err != nil { + return err + } for _, field := range fields { - writeField(element, "", field) + if err := writeField(tmpBlock, element, "", field); err != nil { + return err + } + } + if _, err := tmpBlock.WriteString("</tr>"); err != nil { + return err } - tmpBlock.WriteString("</tr>") row++ } - tmpBlock.WriteString("</table>") - - return tmpBlock.Bytes() + if _, err = tmpBlock.WriteString("</table>"); err != nil { + return err + } + return tmpBlock.Flush() } diff --git a/modules/markup/csv/csv_test.go b/modules/markup/csv/csv_test.go index 5438ebdf5c..613762f86c 100644 --- a/modules/markup/csv/csv_test.go +++ b/modules/markup/csv/csv_test.go @@ -5,13 +5,16 @@ package markup import ( + "strings" "testing" + "code.gitea.io/gitea/modules/markup" + "github.com/stretchr/testify/assert" ) func TestRenderCSV(t *testing.T) { - var parser Parser + var render Renderer var kases = map[string]string{ "a": "<table class=\"data-table\"><tr><th class=\"line-num\">1</th><th>a</th></tr></table>", "1,2": "<table class=\"data-table\"><tr><th class=\"line-num\">1</th><th>1</th><th>2</th></tr></table>", @@ -20,7 +23,9 @@ func TestRenderCSV(t *testing.T) { } for k, v := range kases { - res := parser.Render([]byte(k), "", nil, false) - assert.EqualValues(t, v, string(res)) + var buf strings.Builder + err := render.Render(&markup.RenderContext{}, strings.NewReader(k), &buf) + assert.NoError(t, err) + assert.EqualValues(t, v, buf.String()) } } diff --git a/modules/markup/external/external.go b/modules/markup/external/external.go index 6e7e59970d..62814c9914 100644 --- a/modules/markup/external/external.go +++ b/modules/markup/external/external.go @@ -5,7 +5,7 @@ package external import ( - "bytes" + "fmt" "io" "io/ioutil" "os" @@ -19,32 +19,32 @@ import ( "code.gitea.io/gitea/modules/util" ) -// RegisterParsers registers all supported third part parsers according settings -func RegisterParsers() { - for _, parser := range setting.ExternalMarkupParsers { - if parser.Enabled && parser.Command != "" && len(parser.FileExtensions) > 0 { - markup.RegisterParser(&Parser{parser}) +// RegisterRenderers registers all supported third part renderers according settings +func RegisterRenderers() { + for _, renderer := range setting.ExternalMarkupRenderers { + if renderer.Enabled && renderer.Command != "" && len(renderer.FileExtensions) > 0 { + markup.RegisterRenderer(&Renderer{renderer}) } } } -// Parser implements markup.Parser for external tools -type Parser struct { - setting.MarkupParser +// Renderer implements markup.Renderer for external tools +type Renderer struct { + setting.MarkupRenderer } // Name returns the external tool name -func (p *Parser) Name() string { +func (p *Renderer) Name() string { return p.MarkupName } -// NeedPostProcess implements markup.Parser -func (p *Parser) NeedPostProcess() bool { - return p.MarkupParser.NeedPostProcess +// NeedPostProcess implements markup.Renderer +func (p *Renderer) NeedPostProcess() bool { + return p.MarkupRenderer.NeedPostProcess } // Extensions returns the supported extensions of the tool -func (p *Parser) Extensions() []string { +func (p *Renderer) Extensions() []string { return p.FileExtensions } @@ -56,14 +56,10 @@ func envMark(envName string) string { } // Render renders the data of the document to HTML via the external tool. -func (p *Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte { +func (p *Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { var ( - bs []byte - buf = bytes.NewBuffer(bs) - rd = bytes.NewReader(rawBytes) - urlRawPrefix = strings.Replace(urlPrefix, "/src/", "/raw/", 1) - - command = strings.NewReplacer(envMark("GITEA_PREFIX_SRC"), urlPrefix, + urlRawPrefix = strings.Replace(ctx.URLPrefix, "/src/", "/raw/", 1) + command = strings.NewReplacer(envMark("GITEA_PREFIX_SRC"), ctx.URLPrefix, envMark("GITEA_PREFIX_RAW"), urlRawPrefix).Replace(p.Command) commands = strings.Fields(command) args = commands[1:] @@ -73,8 +69,7 @@ func (p *Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]stri // write to temp file f, err := ioutil.TempFile("", "gitea_input") if err != nil { - log.Error("%s create temp file when rendering %s failed: %v", p.Name(), p.Command, err) - return []byte("") + return fmt.Errorf("%s create temp file when rendering %s failed: %v", p.Name(), p.Command, err) } tmpPath := f.Name() defer func() { @@ -83,17 +78,15 @@ func (p *Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]stri } }() - _, err = io.Copy(f, rd) + _, err = io.Copy(f, input) if err != nil { f.Close() - log.Error("%s write data to temp file when rendering %s failed: %v", p.Name(), p.Command, err) - return []byte("") + return fmt.Errorf("%s write data to temp file when rendering %s failed: %v", p.Name(), p.Command, err) } err = f.Close() if err != nil { - log.Error("%s close temp file when rendering %s failed: %v", p.Name(), p.Command, err) - return []byte("") + return fmt.Errorf("%s close temp file when rendering %s failed: %v", p.Name(), p.Command, err) } args = append(args, f.Name()) } @@ -101,16 +94,15 @@ func (p *Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]stri cmd := exec.Command(commands[0], args...) cmd.Env = append( os.Environ(), - "GITEA_PREFIX_SRC="+urlPrefix, + "GITEA_PREFIX_SRC="+ctx.URLPrefix, "GITEA_PREFIX_RAW="+urlRawPrefix, ) if !p.IsInputFile { - cmd.Stdin = rd + cmd.Stdin = input } - cmd.Stdout = buf + cmd.Stdout = output if err := cmd.Run(); err != nil { - log.Error("%s render run command %s %v failed: %v", p.Name(), commands[0], args, err) - return []byte("") + return fmt.Errorf("%s render run command %s %v failed: %v", p.Name(), commands[0], args, err) } - return buf.Bytes() + return nil } diff --git a/modules/markup/html.go b/modules/markup/html.go index bec9ba2fb4..7c4c10ee22 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -7,6 +7,8 @@ package markup import ( "bytes" "fmt" + "io" + "io/ioutil" "net/url" "path" "path/filepath" @@ -144,7 +146,7 @@ func (p *postProcessError) Error() string { return "PostProcess: " + p.context + ", " + p.err.Error() } -type processor func(ctx *postProcessCtx, node *html.Node) +type processor func(ctx *RenderContext, node *html.Node) var defaultProcessors = []processor{ fullIssuePatternProcessor, @@ -159,34 +161,17 @@ var defaultProcessors = []processor{ emojiShortCodeProcessor, } -type postProcessCtx struct { - metas map[string]string - urlPrefix string - isWikiMarkdown bool - - // processors used by this context. - procs []processor -} - // PostProcess does the final required transformations to the passed raw HTML // data, and ensures its validity. Transformations include: replacing links and // emails with HTML links, parsing shortlinks in the format of [[Link]], like // MediaWiki, linking issues in the format #ID, and mentions in the format // @user, and others. func PostProcess( - rawHTML []byte, - urlPrefix string, - metas map[string]string, - isWikiMarkdown bool, -) ([]byte, error) { - // create the context from the parameters - ctx := &postProcessCtx{ - metas: metas, - urlPrefix: urlPrefix, - isWikiMarkdown: isWikiMarkdown, - procs: defaultProcessors, - } - return ctx.postProcess(rawHTML) + ctx *RenderContext, + input io.Reader, + output io.Writer, +) error { + return postProcess(ctx, defaultProcessors, input, output) } var commitMessageProcessors = []processor{ @@ -205,23 +190,18 @@ var commitMessageProcessors = []processor{ // the shortLinkProcessor and will add a defaultLinkProcessor if defaultLink is // set, which changes every text node into a link to the passed default link. func RenderCommitMessage( - rawHTML []byte, - urlPrefix, defaultLink string, - metas map[string]string, -) ([]byte, error) { - ctx := &postProcessCtx{ - metas: metas, - urlPrefix: urlPrefix, - procs: commitMessageProcessors, - } - if defaultLink != "" { + ctx *RenderContext, + content string, +) (string, error) { + var procs = commitMessageProcessors + if ctx.DefaultLink != "" { // we don't have to fear data races, because being // commitMessageProcessors of fixed len and cap, every time we append // something to it the slice is realloc+copied, so append always // generates the slice ex-novo. - ctx.procs = append(ctx.procs, genDefaultLinkProcessor(defaultLink)) + procs = append(procs, genDefaultLinkProcessor(ctx.DefaultLink)) } - return ctx.postProcess(rawHTML) + return renderProcessString(ctx, procs, content) } var commitMessageSubjectProcessors = []processor{ @@ -245,83 +225,72 @@ var emojiProcessors = []processor{ // emailAddressProcessor, will add a defaultLinkProcessor if defaultLink is set, // which changes every text node into a link to the passed default link. func RenderCommitMessageSubject( - rawHTML []byte, - urlPrefix, defaultLink string, - metas map[string]string, -) ([]byte, error) { - ctx := &postProcessCtx{ - metas: metas, - urlPrefix: urlPrefix, - procs: commitMessageSubjectProcessors, - } - if defaultLink != "" { + ctx *RenderContext, + content string, +) (string, error) { + var procs = commitMessageSubjectProcessors + if ctx.DefaultLink != "" { // we don't have to fear data races, because being // commitMessageSubjectProcessors of fixed len and cap, every time we // append something to it the slice is realloc+copied, so append always // generates the slice ex-novo. - ctx.procs = append(ctx.procs, genDefaultLinkProcessor(defaultLink)) + procs = append(procs, genDefaultLinkProcessor(ctx.DefaultLink)) } - return ctx.postProcess(rawHTML) + return renderProcessString(ctx, procs, content) } // RenderIssueTitle to process title on individual issue/pull page func RenderIssueTitle( - rawHTML []byte, - urlPrefix string, - metas map[string]string, -) ([]byte, error) { - ctx := &postProcessCtx{ - metas: metas, - urlPrefix: urlPrefix, - procs: []processor{ - issueIndexPatternProcessor, - sha1CurrentPatternProcessor, - emojiShortCodeProcessor, - emojiProcessor, - }, + ctx *RenderContext, + title string, +) (string, error) { + return renderProcessString(ctx, []processor{ + issueIndexPatternProcessor, + sha1CurrentPatternProcessor, + emojiShortCodeProcessor, + emojiProcessor, + }, title) +} + +func renderProcessString(ctx *RenderContext, procs []processor, content string) (string, error) { + var buf strings.Builder + if err := postProcess(ctx, procs, strings.NewReader(content), &buf); err != nil { + return "", err } - return ctx.postProcess(rawHTML) + return buf.String(), nil } // RenderDescriptionHTML will use similar logic as PostProcess, but will // use a single special linkProcessor. func RenderDescriptionHTML( - rawHTML []byte, - urlPrefix string, - metas map[string]string, -) ([]byte, error) { - ctx := &postProcessCtx{ - metas: metas, - urlPrefix: urlPrefix, - procs: []processor{ - descriptionLinkProcessor, - emojiShortCodeProcessor, - emojiProcessor, - }, - } - return ctx.postProcess(rawHTML) + ctx *RenderContext, + content string, +) (string, error) { + return renderProcessString(ctx, []processor{ + descriptionLinkProcessor, + emojiShortCodeProcessor, + emojiProcessor, + }, content) } // RenderEmoji for when we want to just process emoji and shortcodes // in various places it isn't already run through the normal markdown procesor func RenderEmoji( - rawHTML []byte, -) ([]byte, error) { - ctx := &postProcessCtx{ - procs: emojiProcessors, - } - return ctx.postProcess(rawHTML) + content string, +) (string, error) { + return renderProcessString(&RenderContext{}, emojiProcessors, content) } var tagCleaner = regexp.MustCompile(`<((?:/?\w+/\w+)|(?:/[\w ]+/)|(/?[hH][tT][mM][lL]\b)|(/?[hH][eE][aA][dD]\b))`) var nulCleaner = strings.NewReplacer("\000", "") -func (ctx *postProcessCtx) postProcess(rawHTML []byte) ([]byte, error) { - if ctx.procs == nil { - ctx.procs = defaultProcessors +func postProcess(ctx *RenderContext, procs []processor, input io.Reader, output io.Writer) error { + // FIXME: don't read all content to memory + rawHTML, err := ioutil.ReadAll(input) + if err != nil { + return err } - // give a generous extra 50 bytes res := bytes.NewBuffer(make([]byte, 0, len(rawHTML)+50)) // prepend "<html><body>" _, _ = res.WriteString("<html><body>") @@ -335,11 +304,11 @@ func (ctx *postProcessCtx) postProcess(rawHTML []byte) ([]byte, error) { // parse the HTML nodes, err := html.ParseFragment(res, nil) if err != nil { - return nil, &postProcessError{"invalid HTML", err} + return &postProcessError{"invalid HTML", err} } for _, node := range nodes { - ctx.visitNode(node, true) + visitNode(ctx, procs, node, true) } newNodes := make([]*html.Node, 0, len(nodes)) @@ -365,25 +334,17 @@ func (ctx *postProcessCtx) postProcess(rawHTML []byte) ([]byte, error) { } } - nodes = newNodes - - // Create buffer in which the data will be placed again. We know that the - // length will be at least that of res; to spare a few alloc+copy, we - // reuse res, resetting its length to 0. - res.Reset() // Render everything to buf. - for _, node := range nodes { - err = html.Render(res, node) + for _, node := range newNodes { + err = html.Render(output, node) if err != nil { - return nil, &postProcessError{"error rendering processed HTML", err} + return &postProcessError{"error rendering processed HTML", err} } } - - // Everything done successfully, return parsed data. - return res.Bytes(), nil + return nil } -func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) { +func visitNode(ctx *RenderContext, procs []processor, node *html.Node, visitText bool) { // Add user-content- to IDs if they don't already have them for idx, attr := range node.Attr { if attr.Key == "id" && !(strings.HasPrefix(attr.Val, "user-content-") || blackfridayExtRegex.MatchString(attr.Val)) { @@ -399,7 +360,7 @@ func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) { switch node.Type { case html.TextNode: if visitText { - ctx.textNode(node) + textNode(ctx, procs, node) } case html.ElementNode: if node.Data == "img" { @@ -410,8 +371,8 @@ func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) { } link := []byte(attr.Val) if len(link) > 0 && !IsLink(link) { - prefix := ctx.urlPrefix - if ctx.isWikiMarkdown { + prefix := ctx.URLPrefix + if ctx.IsWiki { prefix = util.URLJoin(prefix, "wiki", "raw") } prefix = strings.Replace(prefix, "/src/", "/media/", 1) @@ -449,7 +410,7 @@ func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) { } } for n := node.FirstChild; n != nil; n = n.NextSibling { - ctx.visitNode(n, visitText) + visitNode(ctx, procs, n, visitText) } } // ignore everything else @@ -457,8 +418,8 @@ func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) { // textNode runs the passed node through various processors, in order to handle // all kinds of special links handled by the post-processing. -func (ctx *postProcessCtx) textNode(node *html.Node) { - for _, processor := range ctx.procs { +func textNode(ctx *RenderContext, procs []processor, node *html.Node) { + for _, processor := range procs { processor(ctx, node) } } @@ -609,7 +570,7 @@ func replaceContentList(node *html.Node, i, j int, newNodes []*html.Node) { } } -func mentionProcessor(ctx *postProcessCtx, node *html.Node) { +func mentionProcessor(ctx *RenderContext, node *html.Node) { // We replace only the first mention; other mentions will be addressed later found, loc := references.FindFirstMentionBytes([]byte(node.Data)) if !found { @@ -617,26 +578,26 @@ func mentionProcessor(ctx *postProcessCtx, node *html.Node) { } mention := node.Data[loc.Start:loc.End] var teams string - teams, ok := ctx.metas["teams"] + teams, ok := ctx.Metas["teams"] // FIXME: util.URLJoin may not be necessary here: // - setting.AppURL is defined to have a terminal '/' so unless mention[1:] // is an AppSubURL link we can probably fallback to concatenation. // team mention should follow @orgName/teamName style if ok && strings.Contains(mention, "/") { mentionOrgAndTeam := strings.Split(mention, "/") - if mentionOrgAndTeam[0][1:] == ctx.metas["org"] && strings.Contains(teams, ","+strings.ToLower(mentionOrgAndTeam[1])+",") { - replaceContent(node, loc.Start, loc.End, createLink(util.URLJoin(setting.AppURL, "org", ctx.metas["org"], "teams", mentionOrgAndTeam[1]), mention, "mention")) + if mentionOrgAndTeam[0][1:] == ctx.Metas["org"] && strings.Contains(teams, ","+strings.ToLower(mentionOrgAndTeam[1])+",") { + replaceContent(node, loc.Start, loc.End, createLink(util.URLJoin(setting.AppURL, "org", ctx.Metas["org"], "teams", mentionOrgAndTeam[1]), mention, "mention")) } return } replaceContent(node, loc.Start, loc.End, createLink(util.URLJoin(setting.AppURL, mention[1:]), mention, "mention")) } -func shortLinkProcessor(ctx *postProcessCtx, node *html.Node) { +func shortLinkProcessor(ctx *RenderContext, node *html.Node) { shortLinkProcessorFull(ctx, node, false) } -func shortLinkProcessorFull(ctx *postProcessCtx, node *html.Node, noLink bool) { +func shortLinkProcessorFull(ctx *RenderContext, node *html.Node, noLink bool) { m := shortLinkPattern.FindStringSubmatchIndex(node.Data) if m == nil { return @@ -741,13 +702,13 @@ func shortLinkProcessorFull(ctx *postProcessCtx, node *html.Node, noLink bool) { link = url.PathEscape(link) } } - urlPrefix := ctx.urlPrefix + urlPrefix := ctx.URLPrefix if image { if !absoluteLink { if IsSameDomain(urlPrefix) { urlPrefix = strings.Replace(urlPrefix, "/src/", "/raw/", 1) } - if ctx.isWikiMarkdown { + if ctx.IsWiki { link = util.URLJoin("wiki", "raw", link) } link = util.URLJoin(urlPrefix, link) @@ -778,7 +739,7 @@ func shortLinkProcessorFull(ctx *postProcessCtx, node *html.Node, noLink bool) { } } else { if !absoluteLink { - if ctx.isWikiMarkdown { + if ctx.IsWiki { link = util.URLJoin("wiki", link) } link = util.URLJoin(urlPrefix, link) @@ -794,8 +755,8 @@ func shortLinkProcessorFull(ctx *postProcessCtx, node *html.Node, noLink bool) { replaceContent(node, m[0], m[1], linkNode) } -func fullIssuePatternProcessor(ctx *postProcessCtx, node *html.Node) { - if ctx.metas == nil { +func fullIssuePatternProcessor(ctx *RenderContext, node *html.Node) { + if ctx.Metas == nil { return } m := getIssueFullPattern().FindStringSubmatchIndex(node.Data) @@ -811,7 +772,7 @@ func fullIssuePatternProcessor(ctx *postProcessCtx, node *html.Node) { matchOrg := linkParts[len(linkParts)-4] matchRepo := linkParts[len(linkParts)-3] - if matchOrg == ctx.metas["user"] && matchRepo == ctx.metas["repo"] { + if matchOrg == ctx.Metas["user"] && matchRepo == ctx.Metas["repo"] { // TODO if m[4]:m[5] is not nil, then link is to a comment, // and we should indicate that in the text somehow replaceContent(node, m[0], m[1], createLink(link, id, "ref-issue")) @@ -822,8 +783,8 @@ func fullIssuePatternProcessor(ctx *postProcessCtx, node *html.Node) { } } -func issueIndexPatternProcessor(ctx *postProcessCtx, node *html.Node) { - if ctx.metas == nil { +func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) { + if ctx.Metas == nil { return } @@ -832,8 +793,8 @@ func issueIndexPatternProcessor(ctx *postProcessCtx, node *html.Node) { ref *references.RenderizableReference ) - _, exttrack := ctx.metas["format"] - alphanum := ctx.metas["style"] == IssueNameStyleAlphanumeric + _, exttrack := ctx.Metas["format"] + alphanum := ctx.Metas["style"] == IssueNameStyleAlphanumeric // Repos with external issue trackers might still need to reference local PRs // We need to concern with the first one that shows up in the text, whichever it is @@ -853,8 +814,8 @@ func issueIndexPatternProcessor(ctx *postProcessCtx, node *html.Node) { var link *html.Node reftext := node.Data[ref.RefLocation.Start:ref.RefLocation.End] if exttrack && !ref.IsPull { - ctx.metas["index"] = ref.Issue - link = createLink(com.Expand(ctx.metas["format"], ctx.metas), reftext, "ref-issue") + ctx.Metas["index"] = ref.Issue + link = createLink(com.Expand(ctx.Metas["format"], ctx.Metas), reftext, "ref-issue") } else { // Path determines the type of link that will be rendered. It's unknown at this point whether // the linked item is actually a PR or an issue. Luckily it's of no real consequence because @@ -864,7 +825,7 @@ func issueIndexPatternProcessor(ctx *postProcessCtx, node *html.Node) { path = "pulls" } if ref.Owner == "" { - link = createLink(util.URLJoin(setting.AppURL, ctx.metas["user"], ctx.metas["repo"], path, ref.Issue), reftext, "ref-issue") + link = createLink(util.URLJoin(setting.AppURL, ctx.Metas["user"], ctx.Metas["repo"], path, ref.Issue), reftext, "ref-issue") } else { link = createLink(util.URLJoin(setting.AppURL, ref.Owner, ref.Name, path, ref.Issue), reftext, "ref-issue") } @@ -893,8 +854,8 @@ func issueIndexPatternProcessor(ctx *postProcessCtx, node *html.Node) { } // fullSha1PatternProcessor renders SHA containing URLs -func fullSha1PatternProcessor(ctx *postProcessCtx, node *html.Node) { - if ctx.metas == nil { +func fullSha1PatternProcessor(ctx *RenderContext, node *html.Node) { + if ctx.Metas == nil { return } m := anySHA1Pattern.FindStringSubmatchIndex(node.Data) @@ -944,8 +905,7 @@ func fullSha1PatternProcessor(ctx *postProcessCtx, node *html.Node) { } // emojiShortCodeProcessor for rendering text like :smile: into emoji -func emojiShortCodeProcessor(ctx *postProcessCtx, node *html.Node) { - +func emojiShortCodeProcessor(ctx *RenderContext, node *html.Node) { m := EmojiShortCodeRegex.FindStringSubmatchIndex(node.Data) if m == nil { return @@ -968,7 +928,7 @@ func emojiShortCodeProcessor(ctx *postProcessCtx, node *html.Node) { } // emoji processor to match emoji and add emoji class -func emojiProcessor(ctx *postProcessCtx, node *html.Node) { +func emojiProcessor(ctx *RenderContext, node *html.Node) { m := emoji.FindEmojiSubmatchIndex(node.Data) if m == nil { return @@ -983,8 +943,8 @@ func emojiProcessor(ctx *postProcessCtx, node *html.Node) { // sha1CurrentPatternProcessor renders SHA1 strings to corresponding links that // are assumed to be in the same repository. -func sha1CurrentPatternProcessor(ctx *postProcessCtx, node *html.Node) { - if ctx.metas == nil || ctx.metas["user"] == "" || ctx.metas["repo"] == "" || ctx.metas["repoPath"] == "" { +func sha1CurrentPatternProcessor(ctx *RenderContext, node *html.Node) { + if ctx.Metas == nil || ctx.Metas["user"] == "" || ctx.Metas["repo"] == "" || ctx.Metas["repoPath"] == "" { return } m := sha1CurrentPattern.FindStringSubmatchIndex(node.Data) @@ -1000,7 +960,7 @@ func sha1CurrentPatternProcessor(ctx *postProcessCtx, node *html.Node) { // as used by git and github for linking and thus we have to do similar. // Because of this, we check to make sure that a matched hash is actually // a commit in the repository before making it a link. - if _, err := git.NewCommand("rev-parse", "--verify", hash).RunInDirBytes(ctx.metas["repoPath"]); err != nil { + if _, err := git.NewCommand("rev-parse", "--verify", hash).RunInDirBytes(ctx.Metas["repoPath"]); err != nil { if !strings.Contains(err.Error(), "fatal: Needed a single revision") { log.Debug("sha1CurrentPatternProcessor git rev-parse: %v", err) } @@ -1008,11 +968,11 @@ func sha1CurrentPatternProcessor(ctx *postProcessCtx, node *html.Node) { } replaceContent(node, m[2], m[3], - createCodeLink(util.URLJoin(setting.AppURL, ctx.metas["user"], ctx.metas["repo"], "commit", hash), base.ShortSha(hash), "commit")) + createCodeLink(util.URLJoin(setting.AppURL, ctx.Metas["user"], ctx.Metas["repo"], "commit", hash), base.ShortSha(hash), "commit")) } // emailAddressProcessor replaces raw email addresses with a mailto: link. -func emailAddressProcessor(ctx *postProcessCtx, node *html.Node) { +func emailAddressProcessor(ctx *RenderContext, node *html.Node) { m := emailRegex.FindStringSubmatchIndex(node.Data) if m == nil { return @@ -1023,7 +983,7 @@ func emailAddressProcessor(ctx *postProcessCtx, node *html.Node) { // linkProcessor creates links for any HTTP or HTTPS URL not captured by // markdown. -func linkProcessor(ctx *postProcessCtx, node *html.Node) { +func linkProcessor(ctx *RenderContext, node *html.Node) { m := common.LinkRegex.FindStringIndex(node.Data) if m == nil { return @@ -1033,7 +993,7 @@ func linkProcessor(ctx *postProcessCtx, node *html.Node) { } func genDefaultLinkProcessor(defaultLink string) processor { - return func(ctx *postProcessCtx, node *html.Node) { + return func(ctx *RenderContext, node *html.Node) { ch := &html.Node{ Parent: node, Type: html.TextNode, @@ -1052,7 +1012,7 @@ func genDefaultLinkProcessor(defaultLink string) processor { } // descriptionLinkProcessor creates links for DescriptionHTML -func descriptionLinkProcessor(ctx *postProcessCtx, node *html.Node) { +func descriptionLinkProcessor(ctx *RenderContext, node *html.Node) { m := common.LinkRegex.FindStringIndex(node.Data) if m == nil { return diff --git a/modules/markup/html_internal_test.go b/modules/markup/html_internal_test.go index 7e4bb6f22f..330750a47a 100644 --- a/modules/markup/html_internal_test.go +++ b/modules/markup/html_internal_test.go @@ -61,8 +61,8 @@ var localMetas = map[string]string{ func TestRender_IssueIndexPattern(t *testing.T) { // numeric: render inputs without valid mentions test := func(s string) { - testRenderIssueIndexPattern(t, s, s, nil) - testRenderIssueIndexPattern(t, s, s, &postProcessCtx{metas: numericMetas}) + testRenderIssueIndexPattern(t, s, s, &RenderContext{}) + testRenderIssueIndexPattern(t, s, s, &RenderContext{Metas: numericMetas}) } // should not render anything when there are no mentions @@ -109,13 +109,13 @@ func TestRender_IssueIndexPattern2(t *testing.T) { links[i] = numericIssueLink(util.URLJoin(setting.AppSubURL, path), "ref-issue", index, marker) } expectedNil := fmt.Sprintf(expectedFmt, links...) - testRenderIssueIndexPattern(t, s, expectedNil, &postProcessCtx{metas: localMetas}) + testRenderIssueIndexPattern(t, s, expectedNil, &RenderContext{Metas: localMetas}) for i, index := range indices { links[i] = numericIssueLink(prefix, "ref-issue", index, marker) } expectedNum := fmt.Sprintf(expectedFmt, links...) - testRenderIssueIndexPattern(t, s, expectedNum, &postProcessCtx{metas: numericMetas}) + testRenderIssueIndexPattern(t, s, expectedNum, &RenderContext{Metas: numericMetas}) } // should render freestanding mentions @@ -150,7 +150,7 @@ func TestRender_IssueIndexPattern3(t *testing.T) { // alphanumeric: render inputs without valid mentions test := func(s string) { - testRenderIssueIndexPattern(t, s, s, &postProcessCtx{metas: alphanumericMetas}) + testRenderIssueIndexPattern(t, s, s, &RenderContext{Metas: alphanumericMetas}) } test("") test("this is a test") @@ -181,25 +181,22 @@ func TestRender_IssueIndexPattern4(t *testing.T) { links[i] = alphanumIssueLink("https://someurl.com/someUser/someRepo/", "ref-issue", name) } expected := fmt.Sprintf(expectedFmt, links...) - testRenderIssueIndexPattern(t, s, expected, &postProcessCtx{metas: alphanumericMetas}) + testRenderIssueIndexPattern(t, s, expected, &RenderContext{Metas: alphanumericMetas}) } test("OTT-1234 test", "%s test", "OTT-1234") test("test T-12 issue", "test %s issue", "T-12") test("test issue ABCDEFGHIJ-1234567890", "test issue %s", "ABCDEFGHIJ-1234567890") } -func testRenderIssueIndexPattern(t *testing.T, input, expected string, ctx *postProcessCtx) { - if ctx == nil { - ctx = new(postProcessCtx) - } - ctx.procs = []processor{issueIndexPatternProcessor} - if ctx.urlPrefix == "" { - ctx.urlPrefix = AppSubURL +func testRenderIssueIndexPattern(t *testing.T, input, expected string, ctx *RenderContext) { + if ctx.URLPrefix == "" { + ctx.URLPrefix = AppSubURL } - res, err := ctx.postProcess([]byte(input)) + var buf strings.Builder + err := postProcess(ctx, []processor{issueIndexPatternProcessor}, strings.NewReader(input), &buf) assert.NoError(t, err) - assert.Equal(t, expected, string(res)) + assert.Equal(t, expected, buf.String()) } func TestRender_AutoLink(t *testing.T) { @@ -207,12 +204,22 @@ func TestRender_AutoLink(t *testing.T) { setting.AppSubURL = AppSubURL test := func(input, expected string) { - buffer, err := PostProcess([]byte(input), setting.AppSubURL, localMetas, false) + var buffer strings.Builder + err := PostProcess(&RenderContext{ + URLPrefix: setting.AppSubURL, + Metas: localMetas, + }, strings.NewReader(input), &buffer) assert.Equal(t, err, nil) - assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer))) - buffer, err = PostProcess([]byte(input), setting.AppSubURL, localMetas, true) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer.String())) + + buffer.Reset() + err = PostProcess(&RenderContext{ + URLPrefix: setting.AppSubURL, + Metas: localMetas, + IsWiki: true, + }, strings.NewReader(input), &buffer) assert.Equal(t, err, nil) - assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer))) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer.String())) } // render valid issue URLs @@ -235,15 +242,13 @@ func TestRender_FullIssueURLs(t *testing.T) { setting.AppSubURL = AppSubURL test := func(input, expected string) { - ctx := new(postProcessCtx) - ctx.procs = []processor{fullIssuePatternProcessor} - if ctx.urlPrefix == "" { - ctx.urlPrefix = AppSubURL - } - ctx.metas = localMetas - result, err := ctx.postProcess([]byte(input)) + var result strings.Builder + err := postProcess(&RenderContext{ + URLPrefix: AppSubURL, + Metas: localMetas, + }, []processor{fullIssuePatternProcessor}, strings.NewReader(input), &result) assert.NoError(t, err) - assert.Equal(t, expected, string(result)) + assert.Equal(t, expected, result.String()) } test("Here is a link https://git.osgeo.org/gogs/postgis/postgis/pulls/6", "Here is a link https://git.osgeo.org/gogs/postgis/postgis/pulls/6") diff --git a/modules/markup/html_test.go b/modules/markup/html_test.go index 1e39be401b..3425c3d3a8 100644 --- a/modules/markup/html_test.go +++ b/modules/markup/html_test.go @@ -28,7 +28,12 @@ func TestRender_Commits(t *testing.T) { setting.AppSubURL = AppSubURL test := func(input, expected string) { - buffer := RenderString(".md", input, setting.AppSubURL, localMetas) + buffer, err := RenderString(&RenderContext{ + Filename: ".md", + URLPrefix: setting.AppSubURL, + Metas: localMetas, + }, input) + assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } @@ -59,7 +64,12 @@ func TestRender_CrossReferences(t *testing.T) { setting.AppSubURL = AppSubURL test := func(input, expected string) { - buffer := RenderString("a.md", input, setting.AppSubURL, localMetas) + buffer, err := RenderString(&RenderContext{ + Filename: "a.md", + URLPrefix: setting.AppSubURL, + Metas: localMetas, + }, input) + assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } @@ -91,7 +101,11 @@ func TestRender_links(t *testing.T) { setting.AppSubURL = AppSubURL test := func(input, expected string) { - buffer := RenderString("a.md", input, setting.AppSubURL, nil) + buffer, err := RenderString(&RenderContext{ + Filename: "a.md", + URLPrefix: setting.AppSubURL, + }, input) + assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } // Text that should be turned into URL @@ -187,8 +201,12 @@ func TestRender_email(t *testing.T) { setting.AppSubURL = AppSubURL test := func(input, expected string) { - buffer := RenderString("a.md", input, setting.AppSubURL, nil) - assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) + res, err := RenderString(&RenderContext{ + Filename: "a.md", + URLPrefix: setting.AppSubURL, + }, input) + assert.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(res)) } // Text that should be turned into email link @@ -242,7 +260,11 @@ func TestRender_emoji(t *testing.T) { test := func(input, expected string) { expected = strings.ReplaceAll(expected, "&", "&") - buffer := RenderString("a.md", input, setting.AppSubURL, nil) + buffer, err := RenderString(&RenderContext{ + Filename: "a.md", + URLPrefix: setting.AppSubURL, + }, input) + assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } @@ -291,9 +313,17 @@ func TestRender_ShortLinks(t *testing.T) { tree := util.URLJoin(AppSubURL, "src", "master") test := func(input, expected, expectedWiki string) { - buffer := markdown.RenderString(input, tree, nil) + buffer, err := markdown.RenderString(&RenderContext{ + URLPrefix: tree, + }, input) + assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) - buffer = markdown.RenderWiki([]byte(input), setting.AppSubURL, localMetas) + buffer, err = markdown.RenderString(&RenderContext{ + URLPrefix: setting.AppSubURL, + Metas: localMetas, + IsWiki: true, + }, input) + assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expectedWiki), strings.TrimSpace(buffer)) } @@ -395,16 +425,22 @@ func Test_ParseClusterFuzz(t *testing.T) { data := "<A><maTH><tr><MN><bodY ÿ><temPlate></template><tH><tr></A><tH><d<bodY " - val, err := PostProcess([]byte(data), "https://example.com", localMetas, false) - + var res strings.Builder + err := PostProcess(&RenderContext{ + URLPrefix: "https://example.com", + Metas: localMetas, + }, strings.NewReader(data), &res) assert.NoError(t, err) - assert.NotContains(t, string(val), "<html") + assert.NotContains(t, res.String(), "<html") data = "<!DOCTYPE html>\n<A><maTH><tr><MN><bodY ÿ><temPlate></template><tH><tr></A><tH><d<bodY " - val, err = PostProcess([]byte(data), "https://example.com", localMetas, false) + res.Reset() + err = PostProcess(&RenderContext{ + URLPrefix: "https://example.com", + Metas: localMetas, + }, strings.NewReader(data), &res) assert.NoError(t, err) - - assert.NotContains(t, string(val), "<html") + assert.NotContains(t, res.String(), "<html") } diff --git a/modules/markup/markdown/markdown.go b/modules/markup/markdown/markdown.go index 5bb0fbd652..87fae2a23b 100644 --- a/modules/markup/markdown/markdown.go +++ b/modules/markup/markdown/markdown.go @@ -8,6 +8,7 @@ package markdown import ( "fmt" "io" + "io/ioutil" "strings" "sync" @@ -73,17 +74,17 @@ func (l *limitWriter) CloseWithError(err error) error { return l.w.CloseWithError(err) } -// NewGiteaParseContext creates a parser.Context with the gitea context set -func NewGiteaParseContext(urlPrefix string, metas map[string]string, isWiki bool) parser.Context { +// newParserContext creates a parser.Context with the render context set +func newParserContext(ctx *markup.RenderContext) parser.Context { pc := parser.NewContext(parser.WithIDs(newPrefixedIDs())) - pc.Set(urlPrefixKey, urlPrefix) - pc.Set(isWikiKey, isWiki) - pc.Set(renderMetasKey, metas) + pc.Set(urlPrefixKey, ctx.URLPrefix) + pc.Set(isWikiKey, ctx.IsWiki) + pc.Set(renderMetasKey, ctx.Metas) return pc } // actualRender renders Markdown to HTML without handling special links. -func actualRender(body []byte, urlPrefix string, metas map[string]string, wikiMarkdown bool) []byte { +func actualRender(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { once.Do(func() { converter = goldmark.New( goldmark.WithExtensions(extension.Table, @@ -169,7 +170,7 @@ func actualRender(body []byte, urlPrefix string, metas map[string]string, wikiMa limit: setting.UI.MaxDisplayFileSize * 3, } - // FIXME: should we include a timeout that closes the pipe to abort the parser and sanitizer if it takes too long? + // FIXME: should we include a timeout that closes the pipe to abort the renderer and sanitizer if it takes too long? go func() { defer func() { err := recover() @@ -184,18 +185,26 @@ func actualRender(body []byte, urlPrefix string, metas map[string]string, wikiMa _ = lw.CloseWithError(fmt.Errorf("%v", err)) }() - pc := NewGiteaParseContext(urlPrefix, metas, wikiMarkdown) - if err := converter.Convert(giteautil.NormalizeEOL(body), lw, parser.WithContext(pc)); err != nil { + // FIXME: Don't read all to memory, but goldmark doesn't support + pc := newParserContext(ctx) + buf, err := ioutil.ReadAll(input) + if err != nil { + log.Error("Unable to ReadAll: %v", err) + return + } + if err := converter.Convert(giteautil.NormalizeEOL(buf), lw, parser.WithContext(pc)); err != nil { log.Error("Unable to render: %v", err) _ = lw.CloseWithError(err) return } _ = lw.Close() }() - return markup.SanitizeReader(rd).Bytes() + buf := markup.SanitizeReader(rd) + _, err := io.Copy(output, buf) + return err } -func render(body []byte, urlPrefix string, metas map[string]string, wikiMarkdown bool) (ret []byte) { +func render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { defer func() { err := recover() if err == nil { @@ -206,9 +215,13 @@ func render(body []byte, urlPrefix string, metas map[string]string, wikiMarkdown if log.IsDebug() { log.Debug("Panic in markdown: %v\n%s", err, string(log.Stack(2))) } - ret = markup.SanitizeBytes(body) + ret := markup.SanitizeReader(input) + _, err = io.Copy(output, ret) + if err != nil { + log.Error("SanitizeReader failed: %v", err) + } }() - return actualRender(body, urlPrefix, metas, wikiMarkdown) + return actualRender(ctx, input, output) } var ( @@ -217,48 +230,59 @@ var ( ) func init() { - markup.RegisterParser(Parser{}) + markup.RegisterRenderer(Renderer{}) } -// Parser implements markup.Parser -type Parser struct{} +// Renderer implements markup.Renderer +type Renderer struct{} -// Name implements markup.Parser -func (Parser) Name() string { +// Name implements markup.Renderer +func (Renderer) Name() string { return MarkupName } -// NeedPostProcess implements markup.Parser -func (Parser) NeedPostProcess() bool { return true } +// NeedPostProcess implements markup.Renderer +func (Renderer) NeedPostProcess() bool { return true } -// Extensions implements markup.Parser -func (Parser) Extensions() []string { +// Extensions implements markup.Renderer +func (Renderer) Extensions() []string { return setting.Markdown.FileExtensions } -// Render implements markup.Parser -func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte { - return render(rawBytes, urlPrefix, metas, isWiki) +// Render implements markup.Renderer +func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { + return render(ctx, input, output) } // Render renders Markdown to HTML with all specific handling stuff. -func Render(rawBytes []byte, urlPrefix string, metas map[string]string) []byte { - return markup.Render("a.md", rawBytes, urlPrefix, metas) +func Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { + if ctx.Filename == "" { + ctx.Filename = "a.md" + } + return markup.Render(ctx, input, output) } -// RenderRaw renders Markdown to HTML without handling special links. -func RenderRaw(body []byte, urlPrefix string, wikiMarkdown bool) []byte { - return render(body, urlPrefix, map[string]string{}, wikiMarkdown) +// RenderString renders Markdown string to HTML with all specific handling stuff and return string +func RenderString(ctx *markup.RenderContext, content string) (string, error) { + var buf strings.Builder + if err := Render(ctx, strings.NewReader(content), &buf); err != nil { + return "", err + } + return buf.String(), nil } -// RenderString renders Markdown to HTML with special links and returns string type. -func RenderString(raw, urlPrefix string, metas map[string]string) string { - return markup.RenderString("a.md", raw, urlPrefix, metas) +// RenderRaw renders Markdown to HTML without handling special links. +func RenderRaw(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { + return render(ctx, input, output) } -// RenderWiki renders markdown wiki page to HTML and return HTML string -func RenderWiki(rawBytes []byte, urlPrefix string, metas map[string]string) string { - return markup.RenderWiki("a.md", rawBytes, urlPrefix, metas) +// RenderRawString renders Markdown to HTML without handling special links and return string +func RenderRawString(ctx *markup.RenderContext, content string) (string, error) { + var buf strings.Builder + if err := RenderRaw(ctx, strings.NewReader(content), &buf); err != nil { + return "", err + } + return buf.String(), nil } // IsMarkdownFile reports whether name looks like a Markdown file diff --git a/modules/markup/markdown/markdown_test.go b/modules/markup/markdown/markdown_test.go index 0e340763ae..5997dbccdc 100644 --- a/modules/markup/markdown/markdown_test.go +++ b/modules/markup/markdown/markdown_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + "code.gitea.io/gitea/modules/markup" . "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -31,10 +32,17 @@ func TestRender_StandardLinks(t *testing.T) { setting.AppSubURL = AppSubURL test := func(input, expected, expectedWiki string) { - buffer := RenderString(input, setting.AppSubURL, nil) + buffer, err := RenderString(&markup.RenderContext{ + URLPrefix: setting.AppSubURL, + }, input) + assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) - bufferWiki := RenderWiki([]byte(input), setting.AppSubURL, nil) - assert.Equal(t, strings.TrimSpace(expectedWiki), strings.TrimSpace(bufferWiki)) + + buffer, err = RenderString(&markup.RenderContext{ + URLPrefix: setting.AppSubURL, + IsWiki: true, + }, input) + assert.Equal(t, strings.TrimSpace(expectedWiki), strings.TrimSpace(buffer)) } googleRendered := `<p><a href="https://google.com/" rel="nofollow">https://google.com/</a></p>` @@ -74,7 +82,10 @@ func TestRender_Images(t *testing.T) { setting.AppSubURL = AppSubURL test := func(input, expected string) { - buffer := RenderString(input, setting.AppSubURL, nil) + buffer, err := RenderString(&markup.RenderContext{ + URLPrefix: setting.AppSubURL, + }, input) + assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } @@ -261,7 +272,12 @@ func TestTotal_RenderWiki(t *testing.T) { answers := testAnswers(util.URLJoin(AppSubURL, "wiki/"), util.URLJoin(AppSubURL, "wiki", "raw/")) for i := 0; i < len(sameCases); i++ { - line := RenderWiki([]byte(sameCases[i]), AppSubURL, localMetas) + line, err := RenderString(&markup.RenderContext{ + URLPrefix: AppSubURL, + Metas: localMetas, + IsWiki: true, + }, sameCases[i]) + assert.NoError(t, err) assert.Equal(t, answers[i], line) } @@ -279,7 +295,11 @@ func TestTotal_RenderWiki(t *testing.T) { } for i := 0; i < len(testCases); i += 2 { - line := RenderWiki([]byte(testCases[i]), AppSubURL, nil) + line, err := RenderString(&markup.RenderContext{ + URLPrefix: AppSubURL, + IsWiki: true, + }, testCases[i]) + assert.NoError(t, err) assert.Equal(t, testCases[i+1], line) } } @@ -288,31 +308,40 @@ func TestTotal_RenderString(t *testing.T) { answers := testAnswers(util.URLJoin(AppSubURL, "src", "master/"), util.URLJoin(AppSubURL, "raw", "master/")) for i := 0; i < len(sameCases); i++ { - line := RenderString(sameCases[i], util.URLJoin(AppSubURL, "src", "master/"), localMetas) + line, err := RenderString(&markup.RenderContext{ + URLPrefix: util.URLJoin(AppSubURL, "src", "master/"), + Metas: localMetas, + }, sameCases[i]) + assert.NoError(t, err) assert.Equal(t, answers[i], line) } testCases := []string{} for i := 0; i < len(testCases); i += 2 { - line := RenderString(testCases[i], AppSubURL, nil) + line, err := RenderString(&markup.RenderContext{ + URLPrefix: AppSubURL, + }, testCases[i]) + assert.NoError(t, err) assert.Equal(t, testCases[i+1], line) } } func TestRender_RenderParagraphs(t *testing.T) { test := func(t *testing.T, str string, cnt int) { - unix := []byte(str) - res := string(RenderRaw(unix, "", false)) - assert.Equal(t, strings.Count(res, "<p"), cnt, "Rendered result for unix should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res) - - mac := []byte(strings.ReplaceAll(str, "\n", "\r")) - res = string(RenderRaw(mac, "", false)) - assert.Equal(t, strings.Count(res, "<p"), cnt, "Rendered result for mac should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res) - - dos := []byte(strings.ReplaceAll(str, "\n", "\r\n")) - res = string(RenderRaw(dos, "", false)) - assert.Equal(t, strings.Count(res, "<p"), cnt, "Rendered result for windows should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res) + res, err := RenderRawString(&markup.RenderContext{}, str) + assert.NoError(t, err) + assert.Equal(t, cnt, strings.Count(res, "<p"), "Rendered result for unix should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res) + + mac := strings.ReplaceAll(str, "\n", "\r") + res, err = RenderRawString(&markup.RenderContext{}, mac) + assert.NoError(t, err) + assert.Equal(t, cnt, strings.Count(res, "<p"), "Rendered result for mac should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res) + + dos := strings.ReplaceAll(str, "\n", "\r\n") + res, err = RenderRawString(&markup.RenderContext{}, dos) + assert.NoError(t, err) + assert.Equal(t, cnt, strings.Count(res, "<p"), "Rendered result for windows should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res) } test(t, "\nOne\nTwo\nThree", 1) @@ -337,7 +366,8 @@ func TestMarkdownRenderRaw(t *testing.T) { } for _, testcase := range testcases { - _ = RenderRaw(testcase, "", false) + _, err := RenderRawString(&markup.RenderContext{}, string(testcase)) + assert.NoError(t, err) } } @@ -348,7 +378,8 @@ func TestRenderSiblingImages_Issue12925(t *testing.T) { expected := `<p><a href="/image1" rel="nofollow"><img src="/image1" alt="image1"></a><br> <a href="/image2" rel="nofollow"><img src="/image2" alt="image2"></a></p> ` - res := string(RenderRaw([]byte(testcase), "", false)) + res, err := RenderRawString(&markup.RenderContext{}, testcase) + assert.NoError(t, err) assert.Equal(t, expected, res) } diff --git a/modules/markup/markup.go b/modules/markup/markup.go deleted file mode 100644 index bc35757775..0000000000 --- a/modules/markup/markup.go +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2017 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package markup - -import ( - "path/filepath" - "strings" - - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" -) - -// Init initialize regexps for markdown parsing -func Init() { - getIssueFullPattern() - NewSanitizer() - if len(setting.Markdown.CustomURLSchemes) > 0 { - CustomLinkURLSchemes(setting.Markdown.CustomURLSchemes) - } - - // since setting maybe changed extensions, this will reload all parser extensions mapping - extParsers = make(map[string]Parser) - for _, parser := range parsers { - for _, ext := range parser.Extensions() { - extParsers[strings.ToLower(ext)] = parser - } - } -} - -// Parser defines an interface for parsering markup file to HTML -type Parser interface { - Name() string // markup format name - Extensions() []string - NeedPostProcess() bool - Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte -} - -var ( - extParsers = make(map[string]Parser) - parsers = make(map[string]Parser) -) - -// RegisterParser registers a new markup file parser -func RegisterParser(parser Parser) { - parsers[parser.Name()] = parser - for _, ext := range parser.Extensions() { - extParsers[strings.ToLower(ext)] = parser - } -} - -// GetParserByFileName get parser by filename -func GetParserByFileName(filename string) Parser { - extension := strings.ToLower(filepath.Ext(filename)) - return extParsers[extension] -} - -// GetParserByType returns a parser according type -func GetParserByType(tp string) Parser { - return parsers[tp] -} - -// Render renders markup file to HTML with all specific handling stuff. -func Render(filename string, rawBytes []byte, urlPrefix string, metas map[string]string) []byte { - return renderFile(filename, rawBytes, urlPrefix, metas, false) -} - -// RenderByType renders markup to HTML with special links and returns string type. -func RenderByType(tp string, rawBytes []byte, urlPrefix string, metas map[string]string) []byte { - return renderByType(tp, rawBytes, urlPrefix, metas, false) -} - -// RenderString renders Markdown to HTML with special links and returns string type. -func RenderString(filename string, raw, urlPrefix string, metas map[string]string) string { - return string(renderFile(filename, []byte(raw), urlPrefix, metas, false)) -} - -// RenderWiki renders markdown wiki page to HTML and return HTML string -func RenderWiki(filename string, rawBytes []byte, urlPrefix string, metas map[string]string) string { - return string(renderFile(filename, rawBytes, urlPrefix, metas, true)) -} - -func render(parser Parser, rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte { - result := parser.Render(rawBytes, urlPrefix, metas, isWiki) - if parser.NeedPostProcess() { - var err error - // TODO: one day the error should be returned. - result, err = PostProcess(result, urlPrefix, metas, isWiki) - if err != nil { - log.Error("PostProcess: %v", err) - } - } - return SanitizeBytes(result) -} - -func renderByType(tp string, rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte { - if parser, ok := parsers[tp]; ok { - return render(parser, rawBytes, urlPrefix, metas, isWiki) - } - return nil -} - -func renderFile(filename string, rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte { - extension := strings.ToLower(filepath.Ext(filename)) - if parser, ok := extParsers[extension]; ok { - return render(parser, rawBytes, urlPrefix, metas, isWiki) - } - return nil -} - -// Type returns if markup format via the filename -func Type(filename string) string { - if parser := GetParserByFileName(filename); parser != nil { - return parser.Name() - } - return "" -} - -// IsMarkupFile reports whether file is a markup type file -func IsMarkupFile(name, markup string) bool { - if parser := GetParserByFileName(name); parser != nil { - return parser.Name() == markup - } - return false -} - -// IsReadmeFile reports whether name looks like a README file -// based on its name. If an extension is provided, it will strictly -// match that extension. -// Note that the '.' should be provided in ext, e.g ".md" -func IsReadmeFile(name string, ext ...string) bool { - name = strings.ToLower(name) - if len(ext) > 0 { - return name == "readme"+ext[0] - } - if len(name) < 6 { - return false - } else if len(name) == 6 { - return name == "readme" - } - return name[:7] == "readme." -} diff --git a/modules/markup/orgmode/orgmode.go b/modules/markup/orgmode/orgmode.go index b445b76956..96e67f90cf 100644 --- a/modules/markup/orgmode/orgmode.go +++ b/modules/markup/orgmode/orgmode.go @@ -8,9 +8,9 @@ import ( "bytes" "fmt" "html" + "io" "strings" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/util" @@ -18,58 +18,62 @@ import ( ) func init() { - markup.RegisterParser(Parser{}) + markup.RegisterRenderer(Renderer{}) } -// Parser implements markup.Parser for orgmode -type Parser struct { +// Renderer implements markup.Renderer for orgmode +type Renderer struct { } -// Name implements markup.Parser -func (Parser) Name() string { +// Name implements markup.Renderer +func (Renderer) Name() string { return "orgmode" } -// NeedPostProcess implements markup.Parser -func (Parser) NeedPostProcess() bool { return true } +// NeedPostProcess implements markup.Renderer +func (Renderer) NeedPostProcess() bool { return true } -// Extensions implements markup.Parser -func (Parser) Extensions() []string { +// Extensions implements markup.Renderer +func (Renderer) Extensions() []string { return []string{".org"} } // Render renders orgmode rawbytes to HTML -func Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte { +func Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { htmlWriter := org.NewHTMLWriter() - renderer := &Renderer{ + w := &Writer{ HTMLWriter: htmlWriter, - URLPrefix: urlPrefix, - IsWiki: isWiki, + URLPrefix: ctx.URLPrefix, + IsWiki: ctx.IsWiki, } - htmlWriter.ExtendingWriter = renderer + htmlWriter.ExtendingWriter = w - res, err := org.New().Silent().Parse(bytes.NewReader(rawBytes), "").Write(renderer) + res, err := org.New().Silent().Parse(input, "").Write(w) if err != nil { - log.Error("Panic in orgmode.Render: %v Just returning the rawBytes", err) - return rawBytes + return fmt.Errorf("orgmode.Render failed: %v", err) } - return []byte(res) + _, err = io.Copy(output, strings.NewReader(res)) + return err } -// RenderString reners orgmode string to HTML string -func RenderString(rawContent string, urlPrefix string, metas map[string]string, isWiki bool) string { - return string(Render([]byte(rawContent), urlPrefix, metas, isWiki)) +// RenderString renders orgmode string to HTML string +func RenderString(ctx *markup.RenderContext, content string) (string, error) { + var buf strings.Builder + if err := Render(ctx, strings.NewReader(content), &buf); err != nil { + return "", err + } + return buf.String(), nil } -// Render reners orgmode string to HTML string -func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte { - return Render(rawBytes, urlPrefix, metas, isWiki) +// Render renders orgmode string to HTML string +func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { + return Render(ctx, input, output) } -// Renderer implements org.Writer -type Renderer struct { +// Writer implements org.Writer +type Writer struct { *org.HTMLWriter URLPrefix string IsWiki bool @@ -78,7 +82,7 @@ type Renderer struct { var byteMailto = []byte("mailto:") // WriteRegularLink renders images, links or videos -func (r *Renderer) WriteRegularLink(l org.RegularLink) { +func (r *Writer) WriteRegularLink(l org.RegularLink) { link := []byte(html.EscapeString(l.URL)) if l.Protocol == "file" { link = link[len("file:"):] diff --git a/modules/markup/orgmode/orgmode_test.go b/modules/markup/orgmode/orgmode_test.go index 020a3f592a..da89326e9e 100644 --- a/modules/markup/orgmode/orgmode_test.go +++ b/modules/markup/orgmode/orgmode_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -23,7 +24,10 @@ func TestRender_StandardLinks(t *testing.T) { setting.AppSubURL = AppSubURL test := func(input, expected string) { - buffer := RenderString(input, setting.AppSubURL, nil, false) + buffer, err := RenderString(&markup.RenderContext{ + URLPrefix: setting.AppSubURL, + }, input) + assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } @@ -40,7 +44,10 @@ func TestRender_Images(t *testing.T) { setting.AppSubURL = AppSubURL test := func(input, expected string) { - buffer := RenderString(input, setting.AppSubURL, nil, false) + buffer, err := RenderString(&markup.RenderContext{ + URLPrefix: setting.AppSubURL, + }, input) + assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } diff --git a/modules/markup/renderer.go b/modules/markup/renderer.go new file mode 100644 index 0000000000..7cc81574ba --- /dev/null +++ b/modules/markup/renderer.go @@ -0,0 +1,201 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package markup + +import ( + "context" + "errors" + "fmt" + "io" + "path/filepath" + "strings" + "sync" + + "code.gitea.io/gitea/modules/setting" +) + +// Init initialize regexps for markdown parsing +func Init() { + getIssueFullPattern() + NewSanitizer() + if len(setting.Markdown.CustomURLSchemes) > 0 { + CustomLinkURLSchemes(setting.Markdown.CustomURLSchemes) + } + + // since setting maybe changed extensions, this will reload all renderer extensions mapping + extRenderers = make(map[string]Renderer) + for _, renderer := range renderers { + for _, ext := range renderer.Extensions() { + extRenderers[strings.ToLower(ext)] = renderer + } + } +} + +// RenderContext represents a render context +type RenderContext struct { + Ctx context.Context + Filename string + Type string + IsWiki bool + URLPrefix string + Metas map[string]string + DefaultLink string +} + +// Renderer defines an interface for rendering markup file to HTML +type Renderer interface { + Name() string // markup format name + Extensions() []string + Render(ctx *RenderContext, input io.Reader, output io.Writer) error +} + +var ( + extRenderers = make(map[string]Renderer) + renderers = make(map[string]Renderer) +) + +// RegisterRenderer registers a new markup file renderer +func RegisterRenderer(renderer Renderer) { + renderers[renderer.Name()] = renderer + for _, ext := range renderer.Extensions() { + extRenderers[strings.ToLower(ext)] = renderer + } +} + +// GetRendererByFileName get renderer by filename +func GetRendererByFileName(filename string) Renderer { + extension := strings.ToLower(filepath.Ext(filename)) + return extRenderers[extension] +} + +// GetRendererByType returns a renderer according type +func GetRendererByType(tp string) Renderer { + return renderers[tp] +} + +// Render renders markup file to HTML with all specific handling stuff. +func Render(ctx *RenderContext, input io.Reader, output io.Writer) error { + if ctx.Type != "" { + return renderByType(ctx, input, output) + } else if ctx.Filename != "" { + return renderFile(ctx, input, output) + } + return errors.New("Render options both filename and type missing") +} + +// RenderString renders Markup string to HTML with all specific handling stuff and return string +func RenderString(ctx *RenderContext, content string) (string, error) { + var buf strings.Builder + if err := Render(ctx, strings.NewReader(content), &buf); err != nil { + return "", err + } + return buf.String(), nil +} + +func render(ctx *RenderContext, parser Renderer, input io.Reader, output io.Writer) error { + var wg sync.WaitGroup + var err error + pr, pw := io.Pipe() + defer func() { + _ = pr.Close() + _ = pw.Close() + }() + + pr2, pw2 := io.Pipe() + defer func() { + _ = pr2.Close() + _ = pw2.Close() + }() + + wg.Add(1) + go func() { + buf := SanitizeReader(pr2) + _, err = io.Copy(output, buf) + _ = pr2.Close() + wg.Done() + }() + + wg.Add(1) + go func() { + err = PostProcess(ctx, pr, pw2) + _ = pr.Close() + _ = pw2.Close() + wg.Done() + }() + + if err1 := parser.Render(ctx, input, pw); err1 != nil { + return err1 + } + _ = pw.Close() + + wg.Wait() + return err +} + +// ErrUnsupportedRenderType represents +type ErrUnsupportedRenderType struct { + Type string +} + +func (err ErrUnsupportedRenderType) Error() string { + return fmt.Sprintf("Unsupported render type: %s", err.Type) +} + +func renderByType(ctx *RenderContext, input io.Reader, output io.Writer) error { + if renderer, ok := renderers[ctx.Type]; ok { + return render(ctx, renderer, input, output) + } + return ErrUnsupportedRenderType{ctx.Type} +} + +// ErrUnsupportedRenderExtension represents the error when extension doesn't supported to render +type ErrUnsupportedRenderExtension struct { + Extension string +} + +func (err ErrUnsupportedRenderExtension) Error() string { + return fmt.Sprintf("Unsupported render extension: %s", err.Extension) +} + +func renderFile(ctx *RenderContext, input io.Reader, output io.Writer) error { + extension := strings.ToLower(filepath.Ext(ctx.Filename)) + if renderer, ok := extRenderers[extension]; ok { + return render(ctx, renderer, input, output) + } + return ErrUnsupportedRenderExtension{extension} +} + +// Type returns if markup format via the filename +func Type(filename string) string { + if parser := GetRendererByFileName(filename); parser != nil { + return parser.Name() + } + return "" +} + +// IsMarkupFile reports whether file is a markup type file +func IsMarkupFile(name, markup string) bool { + if parser := GetRendererByFileName(name); parser != nil { + return parser.Name() == markup + } + return false +} + +// IsReadmeFile reports whether name looks like a README file +// based on its name. If an extension is provided, it will strictly +// match that extension. +// Note that the '.' should be provided in ext, e.g ".md" +func IsReadmeFile(name string, ext ...string) bool { + name = strings.ToLower(name) + if len(ext) > 0 { + return name == "readme"+ext[0] + } + if len(name) < 6 { + return false + } else if len(name) == 6 { + return name == "readme" + } + return name[:7] == "readme." +} diff --git a/modules/markup/markup_test.go b/modules/markup/renderer_test.go index 118fa2632b..118fa2632b 100644 --- a/modules/markup/markup_test.go +++ b/modules/markup/renderer_test.go diff --git a/modules/notification/mail/mail.go b/modules/notification/mail/mail.go index 9c000da0f6..eb45409faf 100644 --- a/modules/notification/mail/mail.go +++ b/modules/notification/mail/mail.go @@ -104,14 +104,18 @@ func (m *mailNotifier) NotifyIssueChangeAssignee(doer *models.User, issue *model // mail only sent to added assignees and not self-assignee if !removed && doer.ID != assignee.ID && assignee.EmailNotifications() == models.EmailNotificationsEnabled { ct := fmt.Sprintf("Assigned #%d.", issue.Index) - mailer.SendIssueAssignedMail(issue, doer, ct, comment, []*models.User{assignee}) + if err := mailer.SendIssueAssignedMail(issue, doer, ct, comment, []*models.User{assignee}); err != nil { + log.Error("Error in SendIssueAssignedMail for issue[%d] to assignee[%d]: %v", issue.ID, assignee.ID, err) + } } } func (m *mailNotifier) NotifyPullReviewRequest(doer *models.User, issue *models.Issue, reviewer *models.User, isRequest bool, comment *models.Comment) { if isRequest && doer.ID != reviewer.ID && reviewer.EmailNotifications() == models.EmailNotificationsEnabled { ct := fmt.Sprintf("Requested to review %s.", issue.HTMLURL()) - mailer.SendIssueAssignedMail(issue, doer, ct, comment, []*models.User{reviewer}) + if err := mailer.SendIssueAssignedMail(issue, doer, ct, comment, []*models.User{reviewer}); err != nil { + log.Error("Error in SendIssueAssignedMail for issue[%d] to reviewer[%d]: %v", issue.ID, reviewer.ID, err) + } } } diff --git a/modules/setting/markup.go b/modules/setting/markup.go index 36cba68262..f0849a863a 100644 --- a/modules/setting/markup.go +++ b/modules/setting/markup.go @@ -13,14 +13,14 @@ import ( "gopkg.in/ini.v1" ) -// ExternalMarkupParsers represents the external markup parsers +// ExternalMarkupRenderers represents the external markup renderers var ( - ExternalMarkupParsers []MarkupParser - ExternalSanitizerRules []MarkupSanitizerRule + ExternalMarkupRenderers []MarkupRenderer + ExternalSanitizerRules []MarkupSanitizerRule ) -// MarkupParser defines the external parser configured in ini -type MarkupParser struct { +// MarkupRenderer defines the external parser configured in ini +type MarkupRenderer struct { Enabled bool MarkupName string Command string @@ -124,7 +124,7 @@ func newMarkupRenderer(name string, sec *ini.Section) { return } - ExternalMarkupParsers = append(ExternalMarkupParsers, MarkupParser{ + ExternalMarkupRenderers = append(ExternalMarkupRenderers, MarkupRenderer{ Enabled: sec.Key("ENABLED").MustBool(false), MarkupName: name, FileExtensions: exts, diff --git a/modules/templates/helper.go b/modules/templates/helper.go index 7e33f26209..7b175bfab3 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -665,7 +665,11 @@ func RenderCommitMessageLink(msg, urlPrefix, urlDefault string, metas map[string cleanMsg := template.HTMLEscapeString(msg) // we can safely assume that it will not return any error, since there // shouldn't be any special HTML. - fullMessage, err := markup.RenderCommitMessage([]byte(cleanMsg), urlPrefix, urlDefault, metas) + fullMessage, err := markup.RenderCommitMessage(&markup.RenderContext{ + URLPrefix: urlPrefix, + DefaultLink: urlDefault, + Metas: metas, + }, cleanMsg) if err != nil { log.Error("RenderCommitMessage: %v", err) return "" @@ -692,7 +696,11 @@ func RenderCommitMessageLinkSubject(msg, urlPrefix, urlDefault string, metas map // we can safely assume that it will not return any error, since there // shouldn't be any special HTML. - renderedMessage, err := markup.RenderCommitMessageSubject([]byte(template.HTMLEscapeString(msgLine)), urlPrefix, urlDefault, metas) + renderedMessage, err := markup.RenderCommitMessageSubject(&markup.RenderContext{ + URLPrefix: urlPrefix, + DefaultLink: urlDefault, + Metas: metas, + }, template.HTMLEscapeString(msgLine)) if err != nil { log.Error("RenderCommitMessageSubject: %v", err) return template.HTML("") @@ -714,7 +722,10 @@ func RenderCommitBody(msg, urlPrefix string, metas map[string]string) template.H return template.HTML("") } - renderedMessage, err := markup.RenderCommitMessage([]byte(template.HTMLEscapeString(msgLine)), urlPrefix, "", metas) + renderedMessage, err := markup.RenderCommitMessage(&markup.RenderContext{ + URLPrefix: urlPrefix, + Metas: metas, + }, template.HTMLEscapeString(msgLine)) if err != nil { log.Error("RenderCommitMessage: %v", err) return "" @@ -724,7 +735,10 @@ func RenderCommitBody(msg, urlPrefix string, metas map[string]string) template.H // RenderIssueTitle renders issue/pull title with defined post processors func RenderIssueTitle(text, urlPrefix string, metas map[string]string) template.HTML { - renderedText, err := markup.RenderIssueTitle([]byte(template.HTMLEscapeString(text)), urlPrefix, metas) + renderedText, err := markup.RenderIssueTitle(&markup.RenderContext{ + URLPrefix: urlPrefix, + Metas: metas, + }, template.HTMLEscapeString(text)) if err != nil { log.Error("RenderIssueTitle: %v", err) return template.HTML("") @@ -734,7 +748,7 @@ func RenderIssueTitle(text, urlPrefix string, metas map[string]string) template. // RenderEmoji renders html text with emoji post processors func RenderEmoji(text string) template.HTML { - renderedText, err := markup.RenderEmoji([]byte(template.HTMLEscapeString(text))) + renderedText, err := markup.RenderEmoji(template.HTMLEscapeString(text)) if err != nil { log.Error("RenderEmoji: %v", err) return template.HTML("") @@ -758,7 +772,10 @@ func ReactionToEmoji(reaction string) template.HTML { // RenderNote renders the contents of a git-notes file as a commit message. func RenderNote(msg, urlPrefix string, metas map[string]string) template.HTML { cleanMsg := template.HTMLEscapeString(msg) - fullMessage, err := markup.RenderCommitMessage([]byte(cleanMsg), urlPrefix, "", metas) + fullMessage, err := markup.RenderCommitMessage(&markup.RenderContext{ + URLPrefix: urlPrefix, + Metas: metas, + }, cleanMsg) if err != nil { log.Error("RenderNote: %v", err) return "" diff --git a/routers/api/v1/misc/markdown.go b/routers/api/v1/misc/markdown.go index 5718185309..f1007b7ee2 100644 --- a/routers/api/v1/misc/markdown.go +++ b/routers/api/v1/misc/markdown.go @@ -5,11 +5,11 @@ package misc import ( - "io/ioutil" "net/http" "strings" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" @@ -55,7 +55,6 @@ func Markdown(ctx *context.APIContext) { case "comment": fallthrough case "gfm": - md := []byte(form.Text) urlPrefix := form.Context meta := map[string]string{} if !strings.HasPrefix(setting.AppSubURL+"/", urlPrefix) { @@ -77,22 +76,19 @@ func Markdown(ctx *context.APIContext) { if form.Mode == "gfm" { meta["mode"] = "document" } - if form.Wiki { - _, err := ctx.Write([]byte(markdown.RenderWiki(md, urlPrefix, meta))) - if err != nil { - ctx.InternalServerError(err) - return - } - } else { - _, err := ctx.Write(markdown.Render(md, urlPrefix, meta)) - if err != nil { - ctx.InternalServerError(err) - return - } + + if err := markdown.Render(&markup.RenderContext{ + URLPrefix: urlPrefix, + Metas: meta, + IsWiki: form.Wiki, + }, strings.NewReader(form.Text), ctx.Resp); err != nil { + ctx.InternalServerError(err) + return } default: - _, err := ctx.Write(markdown.RenderRaw([]byte(form.Text), "", false)) - if err != nil { + if err := markdown.RenderRaw(&markup.RenderContext{ + URLPrefix: form.Context, + }, strings.NewReader(form.Text), ctx.Resp); err != nil { ctx.InternalServerError(err) return } @@ -120,14 +116,8 @@ func MarkdownRaw(ctx *context.APIContext) { // "$ref": "#/responses/MarkdownRender" // "422": // "$ref": "#/responses/validationError" - - body, err := ioutil.ReadAll(ctx.Req.Body) - if err != nil { - ctx.Error(http.StatusUnprocessableEntity, "", err) - return - } - _, err = ctx.Write(markdown.RenderRaw(body, "", false)) - if err != nil { + defer ctx.Req.Body.Close() + if err := markdown.RenderRaw(&markup.RenderContext{}, ctx.Req.Body, ctx.Resp); err != nil { ctx.InternalServerError(err) return } diff --git a/routers/init.go b/routers/init.go index f5dbfc87d2..220d87a29d 100644 --- a/routers/init.go +++ b/routers/init.go @@ -143,7 +143,7 @@ func GlobalInit(ctx context.Context) { NewServices() highlight.NewContext() - external.RegisterParsers() + external.RegisterRenderers() markup.Init() if setting.EnableSQLite3 { diff --git a/routers/org/home.go b/routers/org/home.go index 9a40d8be6a..d84ae870ab 100644 --- a/routers/org/home.go +++ b/routers/org/home.go @@ -11,6 +11,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" ) @@ -37,7 +38,15 @@ func Home(ctx *context.Context) { ctx.Data["PageIsUserProfile"] = true ctx.Data["Title"] = org.DisplayName() if len(org.Description) != 0 { - ctx.Data["RenderedDescription"] = string(markdown.Render([]byte(org.Description), ctx.Repo.RepoLink, map[string]string{"mode": "document"})) + desc, err := markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Repo.RepoLink, + Metas: map[string]string{"mode": "document"}, + }, org.Description) + if err != nil { + ctx.ServerError("RenderString", err) + return + } + ctx.Data["RenderedDescription"] = desc } var orderBy models.SearchOrderBy diff --git a/routers/repo/compare.go b/routers/repo/compare.go index 7046f3ecdb..a658374d9b 100644 --- a/routers/repo/compare.go +++ b/routers/repo/compare.go @@ -10,7 +10,6 @@ import ( "errors" "fmt" "html" - "io/ioutil" "net/http" "path" "path/filepath" @@ -117,14 +116,7 @@ func setCsvCompareContext(ctx *context.Context) { } defer reader.Close() - b, err := ioutil.ReadAll(reader) - if err != nil { - return nil, err - } - - b = charset.ToUTF8WithFallback(b) - - return csv_module.CreateReaderAndGuessDelimiter(b), nil + return csv_module.CreateReaderAndGuessDelimiter(charset.ToUTF8WithFallbackReader(reader)) } baseReader, err := csvReaderFromCommit(baseCommit) diff --git a/routers/repo/issue.go b/routers/repo/issue.go index 7471bb65a4..12726cd22c 100644 --- a/routers/repo/issue.go +++ b/routers/repo/issue.go @@ -1131,8 +1131,14 @@ func ViewIssue(ctx *context.Context) { } ctx.Data["IssueWatch"] = iw - issue.RenderedContent = string(markdown.Render([]byte(issue.Content), ctx.Repo.RepoLink, - ctx.Repo.Repository.ComposeMetas())) + issue.RenderedContent, err = markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Repo.RepoLink, + Metas: ctx.Repo.Repository.ComposeMetas(), + }, issue.Content) + if err != nil { + ctx.ServerError("RenderString", err) + return + } repo := ctx.Repo.Repository @@ -1289,9 +1295,14 @@ func ViewIssue(ctx *context.Context) { return } - comment.RenderedContent = string(markdown.Render([]byte(comment.Content), ctx.Repo.RepoLink, - ctx.Repo.Repository.ComposeMetas())) - + comment.RenderedContent, err = markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Repo.RepoLink, + Metas: ctx.Repo.Repository.ComposeMetas(), + }, comment.Content) + if err != nil { + ctx.ServerError("RenderString", err) + return + } // Check tag. tag, ok = marked[comment.PosterID] if ok { @@ -1359,8 +1370,14 @@ func ViewIssue(ctx *context.Context) { } } } else if comment.Type == models.CommentTypeCode || comment.Type == models.CommentTypeReview || comment.Type == models.CommentTypeDismissReview { - comment.RenderedContent = string(markdown.Render([]byte(comment.Content), ctx.Repo.RepoLink, - ctx.Repo.Repository.ComposeMetas())) + comment.RenderedContent, err = markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Repo.RepoLink, + Metas: ctx.Repo.Repository.ComposeMetas(), + }, comment.Content) + if err != nil { + ctx.ServerError("RenderString", err) + return + } if err = comment.LoadReview(); err != nil && !models.IsErrReviewNotExist(err) { ctx.ServerError("LoadReview", err) return @@ -1708,10 +1725,20 @@ func UpdateIssueContent(ctx *context.Context) { files := ctx.QueryStrings("files[]") if err := updateAttachments(issue, files); err != nil { ctx.ServerError("UpdateAttachments", err) + return + } + + content, err := markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Query("context"), + Metas: ctx.Repo.Repository.ComposeMetas(), + }, issue.Content) + if err != nil { + ctx.ServerError("RenderString", err) + return } ctx.JSON(http.StatusOK, map[string]interface{}{ - "content": string(markdown.Render([]byte(issue.Content), ctx.Query("context"), ctx.Repo.Repository.ComposeMetas())), + "content": content, "attachments": attachmentsHTML(ctx, issue.Attachments, issue.Content), }) } @@ -2125,10 +2152,20 @@ func UpdateCommentContent(ctx *context.Context) { files := ctx.QueryStrings("files[]") if err := updateAttachments(comment, files); err != nil { ctx.ServerError("UpdateAttachments", err) + return + } + + content, err := markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Query("context"), + Metas: ctx.Repo.Repository.ComposeMetas(), + }, comment.Content) + if err != nil { + ctx.ServerError("RenderString", err) + return } ctx.JSON(http.StatusOK, map[string]interface{}{ - "content": string(markdown.Render([]byte(comment.Content), ctx.Query("context"), ctx.Repo.Repository.ComposeMetas())), + "content": content, "attachments": attachmentsHTML(ctx, comment.Attachments, comment.Content), }) } diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index 457ffb6aba..3a7ce2e23b 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -296,20 +296,13 @@ func LFSFileGet(ctx *context.Context) { break } - d, _ := ioutil.ReadAll(dataRc) - buf = charset.ToUTF8WithFallback(append(buf, d...)) + buf := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc)) // Building code view blocks with line number on server side. - var fileContent string - if content, err := charset.ToUTF8WithErr(buf); err != nil { - log.Error("ToUTF8WithErr: %v", err) - fileContent = string(buf) - } else { - fileContent = content - } + fileContent, _ := ioutil.ReadAll(buf) var output bytes.Buffer - lines := strings.Split(fileContent, "\n") + lines := strings.Split(string(fileContent), "\n") //Remove blank line at the end of file if len(lines) > 0 && lines[len(lines)-1] == "" { lines = lines[:len(lines)-1] diff --git a/routers/repo/milestone.go b/routers/repo/milestone.go index 5a9d2351bc..bb6b310cbe 100644 --- a/routers/repo/milestone.go +++ b/routers/repo/milestone.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" @@ -84,7 +85,14 @@ func Milestones(ctx *context.Context) { } } for _, m := range miles { - m.RenderedContent = string(markdown.Render([]byte(m.Content), ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas())) + m.RenderedContent, err = markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Repo.RepoLink, + Metas: ctx.Repo.Repository.ComposeMetas(), + }, m.Content) + if err != nil { + ctx.ServerError("RenderString", err) + return + } } ctx.Data["Milestones"] = miles @@ -269,7 +277,14 @@ func MilestoneIssuesAndPulls(ctx *context.Context) { return } - milestone.RenderedContent = string(markdown.Render([]byte(milestone.Content), ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas())) + milestone.RenderedContent, err = markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Repo.RepoLink, + Metas: ctx.Repo.Repository.ComposeMetas(), + }, milestone.Content) + if err != nil { + ctx.ServerError("RenderString", err) + return + } ctx.Data["Title"] = milestone.Name ctx.Data["Milestone"] = milestone diff --git a/routers/repo/projects.go b/routers/repo/projects.go index 96ef2c6c0c..eb0719995c 100644 --- a/routers/repo/projects.go +++ b/routers/repo/projects.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -77,7 +78,14 @@ func Projects(ctx *context.Context) { } for i := range projects { - projects[i].RenderedContent = string(markdown.Render([]byte(projects[i].Description), ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas())) + projects[i].RenderedContent, err = markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Repo.RepoLink, + Metas: ctx.Repo.Repository.ComposeMetas(), + }, projects[i].Description) + if err != nil { + ctx.ServerError("RenderString", err) + return + } } ctx.Data["Projects"] = projects @@ -311,7 +319,14 @@ func ViewProject(ctx *context.Context) { } ctx.Data["LinkedPRs"] = linkedPrsMap - project.RenderedContent = string(markdown.Render([]byte(project.Description), ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas())) + project.RenderedContent, err = markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Repo.RepoLink, + Metas: ctx.Repo.Repository.ComposeMetas(), + }, project.Description) + if err != nil { + ctx.ServerError("RenderString", err) + return + } ctx.Data["CanWriteProjects"] = ctx.Repo.Permission.CanWrite(models.UnitTypeProjects) ctx.Data["Project"] = project diff --git a/routers/repo/release.go b/routers/repo/release.go index 2ebb69b6ab..abce3e9ac1 100644 --- a/routers/repo/release.go +++ b/routers/repo/release.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/convert" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/upload" @@ -132,7 +133,14 @@ func releasesOrTags(ctx *context.Context, isTagList bool) { ctx.ServerError("calReleaseNumCommitsBehind", err) return } - r.Note = markdown.RenderString(r.Note, ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas()) + r.Note, err = markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Repo.RepoLink, + Metas: ctx.Repo.Repository.ComposeMetas(), + }, r.Note) + if err != nil { + ctx.ServerError("RenderString", err) + return + } } ctx.Data["Releases"] = releases @@ -182,7 +190,14 @@ func SingleRelease(ctx *context.Context) { ctx.ServerError("calReleaseNumCommitsBehind", err) return } - release.Note = markdown.RenderString(release.Note, ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas()) + release.Note, err = markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Repo.RepoLink, + Metas: ctx.Repo.Repository.ComposeMetas(), + }, release.Note) + if err != nil { + ctx.ServerError("RenderString", err) + return + } ctx.Data["Releases"] = []*models.Release{release} ctx.HTML(http.StatusOK, tplReleases) diff --git a/routers/repo/view.go b/routers/repo/view.go index a03fd58c8a..10deb7065a 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -324,13 +324,26 @@ func renderDirectory(ctx *context.Context, treeLink string) { ctx.Data["IsTextFile"] = true ctx.Data["FileSize"] = fileSize } else { - d, _ := ioutil.ReadAll(dataRc) - buf = charset.ToUTF8WithFallback(append(buf, d...)) + rd := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc)) if markupType := markup.Type(readmeFile.name); markupType != "" { ctx.Data["IsMarkup"] = true ctx.Data["MarkupType"] = string(markupType) - ctx.Data["FileContent"] = string(markup.Render(readmeFile.name, buf, readmeTreelink, ctx.Repo.Repository.ComposeDocumentMetas())) + var result strings.Builder + err := markup.Render(&markup.RenderContext{ + Filename: readmeFile.name, + URLPrefix: readmeTreelink, + Metas: ctx.Repo.Repository.ComposeDocumentMetas(), + }, rd, &result) + if err != nil { + log.Error("Render failed: %v then fallback", err) + bs, _ := ioutil.ReadAll(rd) + ctx.Data["FileContent"] = strings.ReplaceAll( + gotemplate.HTMLEscapeString(string(bs)), "\n", `<br>`, + ) + } else { + ctx.Data["FileContent"] = result.String() + } } else { ctx.Data["IsRenderedHTML"] = true ctx.Data["FileContent"] = strings.ReplaceAll( @@ -481,21 +494,30 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st break } - d, _ := ioutil.ReadAll(dataRc) - buf = charset.ToUTF8WithFallback(append(buf, d...)) + rd := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc)) readmeExist := markup.IsReadmeFile(blob.Name()) ctx.Data["ReadmeExist"] = readmeExist if markupType := markup.Type(blob.Name()); markupType != "" { ctx.Data["IsMarkup"] = true ctx.Data["MarkupType"] = markupType - ctx.Data["FileContent"] = string(markup.Render(blob.Name(), buf, path.Dir(treeLink), ctx.Repo.Repository.ComposeDocumentMetas())) + var result strings.Builder + err := markup.Render(&markup.RenderContext{ + Filename: blob.Name(), + URLPrefix: path.Dir(treeLink), + Metas: ctx.Repo.Repository.ComposeDocumentMetas(), + }, rd, &result) + if err != nil { + ctx.ServerError("Render", err) + return + } + ctx.Data["FileContent"] = result.String() } else if readmeExist { ctx.Data["IsRenderedHTML"] = true ctx.Data["FileContent"] = strings.ReplaceAll( gotemplate.HTMLEscapeString(string(buf)), "\n", `<br>`, ) } else { - buf = charset.ToUTF8WithFallback(buf) + buf, _ := ioutil.ReadAll(rd) lineNums := linesBytesCount(buf) ctx.Data["NumLines"] = strconv.Itoa(lineNums) ctx.Data["NumLinesSet"] = true @@ -532,11 +554,20 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st } if markupType := markup.Type(blob.Name()); markupType != "" { - d, _ := ioutil.ReadAll(dataRc) - buf = append(buf, d...) + rd := io.MultiReader(bytes.NewReader(buf), dataRc) ctx.Data["IsMarkup"] = true ctx.Data["MarkupType"] = markupType - ctx.Data["FileContent"] = string(markup.Render(blob.Name(), buf, path.Dir(treeLink), ctx.Repo.Repository.ComposeDocumentMetas())) + var result strings.Builder + err := markup.Render(&markup.RenderContext{ + Filename: blob.Name(), + URLPrefix: path.Dir(treeLink), + Metas: ctx.Repo.Repository.ComposeDocumentMetas(), + }, rd, &result) + if err != nil { + ctx.ServerError("Render", err) + return + } + ctx.Data["FileContent"] = result.String() } } diff --git a/routers/repo/wiki.go b/routers/repo/wiki.go index 290e2e8bb2..1bdd06dce5 100644 --- a/routers/repo/wiki.go +++ b/routers/repo/wiki.go @@ -6,6 +6,7 @@ package repo import ( + "bytes" "fmt" "io/ioutil" "net/http" @@ -211,12 +212,34 @@ func renderViewPage(ctx *context.Context) (*git.Repository, *git.TreeEntry) { return nil, nil } - metas := ctx.Repo.Repository.ComposeDocumentMetas() - ctx.Data["content"] = markdown.RenderWiki(data, ctx.Repo.RepoLink, metas) + var rctx = &markup.RenderContext{ + URLPrefix: ctx.Repo.RepoLink, + Metas: ctx.Repo.Repository.ComposeDocumentMetas(), + IsWiki: true, + } + + var buf strings.Builder + if err := markdown.Render(rctx, bytes.NewReader(data), &buf); err != nil { + ctx.ServerError("Render", err) + return nil, nil + } + ctx.Data["content"] = buf.String() + + buf.Reset() + if err := markdown.Render(rctx, bytes.NewReader(sidebarContent), &buf); err != nil { + ctx.ServerError("Render", err) + return nil, nil + } ctx.Data["sidebarPresent"] = sidebarContent != nil - ctx.Data["sidebarContent"] = markdown.RenderWiki(sidebarContent, ctx.Repo.RepoLink, metas) + ctx.Data["sidebarContent"] = buf.String() + + buf.Reset() + if err := markdown.Render(rctx, bytes.NewReader(footerContent), &buf); err != nil { + ctx.ServerError("Render", err) + return nil, nil + } ctx.Data["footerPresent"] = footerContent != nil - ctx.Data["footerContent"] = markdown.RenderWiki(footerContent, ctx.Repo.RepoLink, metas) + ctx.Data["footerContent"] = buf.String() // get commit count - wiki revisions commitsCount, _ := wikiRepo.FileCommitsCount("master", pageFilename) diff --git a/routers/user/home.go b/routers/user/home.go index 584bc019fa..acf73f82fe 100644 --- a/routers/user/home.go +++ b/routers/user/home.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/gitea/modules/context" issue_indexer "code.gitea.io/gitea/modules/indexer/issues" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -267,7 +268,15 @@ func Milestones(ctx *context.Context) { continue } - milestones[i].RenderedContent = string(markdown.Render([]byte(milestones[i].Content), milestones[i].Repo.Link(), milestones[i].Repo.ComposeMetas())) + milestones[i].RenderedContent, err = markdown.RenderString(&markup.RenderContext{ + URLPrefix: milestones[i].Repo.Link(), + Metas: milestones[i].Repo.ComposeMetas(), + }, milestones[i].Content) + if err != nil { + ctx.ServerError("RenderString", err) + return + } + if milestones[i].Repo.IsTimetrackerEnabled() { err := milestones[i].LoadTotalTrackedTime() if err != nil { diff --git a/routers/user/profile.go b/routers/user/profile.go index c24614b108..bb4c0cd5b1 100644 --- a/routers/user/profile.go +++ b/routers/user/profile.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -110,7 +111,15 @@ func Profile(ctx *context.Context) { } if len(ctxUser.Description) != 0 { - ctx.Data["RenderedDescription"] = string(markdown.Render([]byte(ctxUser.Description), ctx.Repo.RepoLink, map[string]string{"mode": "document"})) + content, err := markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Repo.RepoLink, + Metas: map[string]string{"mode": "document"}, + }, ctxUser.Description) + if err != nil { + ctx.ServerError("RenderString", err) + return + } + ctx.Data["RenderedDescription"] = content } showPrivate := ctx.IsSigned && (ctx.User.IsAdmin || ctx.User.ID == ctxUser.ID) diff --git a/services/gitdiff/csv_test.go b/services/gitdiff/csv_test.go index 17edea582c..f3dc0c2a2c 100644 --- a/services/gitdiff/csv_test.go +++ b/services/gitdiff/csv_test.go @@ -95,11 +95,17 @@ func TestCSVDiff(t *testing.T) { var baseReader *csv.Reader if len(c.base) > 0 { - baseReader = csv_module.CreateReaderAndGuessDelimiter([]byte(c.base)) + baseReader, err = csv_module.CreateReaderAndGuessDelimiter(strings.NewReader(c.base)) + if err != nil { + t.Errorf("CreateReaderAndGuessDelimiter failed: %s", err) + } } var headReader *csv.Reader if len(c.head) > 0 { - headReader = csv_module.CreateReaderAndGuessDelimiter([]byte(c.head)) + headReader, err = csv_module.CreateReaderAndGuessDelimiter(strings.NewReader(c.head)) + if err != nil { + t.Errorf("CreateReaderAndGuessDelimiter failed: %s", err) + } } result, err := CreateCsvDiff(diff.Files[0], baseReader, headReader) diff --git a/services/mailer/mail.go b/services/mailer/mail.go index c50795968a..f22140c9f7 100644 --- a/services/mailer/mail.go +++ b/services/mailer/mail.go @@ -174,8 +174,7 @@ func SendCollaboratorMail(u, doer *models.User, repo *models.Repository) { SendAsync(msg) } -func composeIssueCommentMessages(ctx *mailCommentContext, lang string, tos []string, fromMention bool, info string) []*Message { - +func composeIssueCommentMessages(ctx *mailCommentContext, lang string, tos []string, fromMention bool, info string) ([]*Message, error) { var ( subject string link string @@ -199,7 +198,14 @@ func composeIssueCommentMessages(ctx *mailCommentContext, lang string, tos []str } // This is the body of the new issue or comment, not the mail body - body := string(markup.RenderByType(markdown.MarkupName, []byte(ctx.Content), ctx.Issue.Repo.HTMLURL(), ctx.Issue.Repo.ComposeMetas())) + body, err := markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Issue.Repo.HTMLURL(), + Metas: ctx.Issue.Repo.ComposeMetas(), + }, ctx.Content) + if err != nil { + return nil, err + } + actType, actName, tplName := actionToTemplate(ctx.Issue, ctx.ActionType, commentType, reviewType) if actName != "new" { @@ -240,14 +246,13 @@ func composeIssueCommentMessages(ctx *mailCommentContext, lang string, tos []str // TODO: i18n templates? if err := subjectTemplates.ExecuteTemplate(&mailSubject, string(tplName), mailMeta); err == nil { subject = sanitizeSubject(mailSubject.String()) + if subject == "" { + subject = fallback + } } else { log.Error("ExecuteTemplate [%s]: %v", tplName+"/subject", err) } - if subject == "" { - subject = fallback - } - subject = emoji.ReplaceAliases(subject) mailMeta["Subject"] = subject @@ -275,7 +280,7 @@ func composeIssueCommentMessages(ctx *mailCommentContext, lang string, tos []str msgs = append(msgs, msg) } - return msgs + return msgs, nil } func sanitizeSubject(subject string) string { @@ -288,21 +293,26 @@ func sanitizeSubject(subject string) string { } // SendIssueAssignedMail composes and sends issue assigned email -func SendIssueAssignedMail(issue *models.Issue, doer *models.User, content string, comment *models.Comment, recipients []*models.User) { +func SendIssueAssignedMail(issue *models.Issue, doer *models.User, content string, comment *models.Comment, recipients []*models.User) error { langMap := make(map[string][]string) for _, user := range recipients { langMap[user.Language] = append(langMap[user.Language], user.Email) } for lang, tos := range langMap { - SendAsyncs(composeIssueCommentMessages(&mailCommentContext{ + msgs, err := composeIssueCommentMessages(&mailCommentContext{ Issue: issue, Doer: doer, ActionType: models.ActionType(0), Content: content, Comment: comment, - }, lang, tos, false, "issue assigned")) + }, lang, tos, false, "issue assigned") + if err != nil { + return err + } + SendAsyncs(msgs) } + return nil } // actionToTemplate returns the type and name of the action facing the user diff --git a/services/mailer/mail_issue.go b/services/mailer/mail_issue.go index 9786a06f62..bb541d27a0 100644 --- a/services/mailer/mail_issue.go +++ b/services/mailer/mail_issue.go @@ -146,7 +146,11 @@ func mailIssueCommentBatch(ctx *mailCommentContext, users []*models.User, visite // working backwards from the last (possibly) incomplete batch. If len(receivers) can be 0 this // starting condition will need to be changed slightly for i := ((len(receivers) - 1) / MailBatchSize) * MailBatchSize; i >= 0; i -= MailBatchSize { - SendAsyncs(composeIssueCommentMessages(ctx, lang, receivers[i:], fromMention, "issue comments")) + msgs, err := composeIssueCommentMessages(ctx, lang, receivers[i:], fromMention, "issue comments") + if err != nil { + return err + } + SendAsyncs(msgs) receivers = receivers[:i] } } diff --git a/services/mailer/mail_release.go b/services/mailer/mail_release.go index 22efe2f046..1e12fe13ac 100644 --- a/services/mailer/mail_release.go +++ b/services/mailer/mail_release.go @@ -10,6 +10,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/translation" @@ -48,7 +49,15 @@ func MailNewRelease(rel *models.Release) { func mailNewRelease(lang string, tos []string, rel *models.Release) { locale := translation.NewLocale(lang) - rel.RenderedNote = markdown.RenderString(rel.Note, rel.Repo.Link(), rel.Repo.ComposeMetas()) + var err error + rel.RenderedNote, err = markdown.RenderString(&markup.RenderContext{ + URLPrefix: rel.Repo.Link(), + Metas: rel.Repo.ComposeMetas(), + }, rel.Note) + if err != nil { + log.Error("markdown.RenderString(%d): %v", rel.RepoID, err) + return + } subject := locale.Tr("mail.release.new.subject", rel.TagName, rel.Repo.FullName()) mailMeta := map[string]interface{}{ diff --git a/services/mailer/mail_test.go b/services/mailer/mail_test.go index 9eef084408..813e51c0d2 100644 --- a/services/mailer/mail_test.go +++ b/services/mailer/mail_test.go @@ -58,8 +58,9 @@ func TestComposeIssueCommentMessage(t *testing.T) { InitMailRender(stpl, btpl) tos := []string{"test@gitea.com", "test2@gitea.com"} - msgs := composeIssueCommentMessages(&mailCommentContext{Issue: issue, Doer: doer, ActionType: models.ActionCommentIssue, + msgs, err := composeIssueCommentMessages(&mailCommentContext{Issue: issue, Doer: doer, ActionType: models.ActionCommentIssue, Content: "test body", Comment: comment}, "en-US", tos, false, "issue comment") + assert.NoError(t, err) assert.Len(t, msgs, 2) gomailMsg := msgs[0].ToMessage() mailto := gomailMsg.GetHeader("To") @@ -92,8 +93,9 @@ func TestComposeIssueMessage(t *testing.T) { InitMailRender(stpl, btpl) tos := []string{"test@gitea.com", "test2@gitea.com"} - msgs := composeIssueCommentMessages(&mailCommentContext{Issue: issue, Doer: doer, ActionType: models.ActionCreateIssue, + msgs, err := composeIssueCommentMessages(&mailCommentContext{Issue: issue, Doer: doer, ActionType: models.ActionCreateIssue, Content: "test body"}, "en-US", tos, false, "issue create") + assert.NoError(t, err) assert.Len(t, msgs, 2) gomailMsg := msgs[0].ToMessage() @@ -218,7 +220,8 @@ func TestTemplateServices(t *testing.T) { } func testComposeIssueCommentMessage(t *testing.T, ctx *mailCommentContext, tos []string, fromMention bool, info string) *Message { - msgs := composeIssueCommentMessages(ctx, "en-US", tos, fromMention, info) + msgs, err := composeIssueCommentMessages(ctx, "en-US", tos, fromMention, info) + assert.NoError(t, err) assert.Len(t, msgs, 1) return msgs[0] } |