From 9d99f6ab19ac3f97af3ca126720e9075c127a652 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 20 Apr 2021 06:25:08 +0800 Subject: Refactor renders (#15175) * Refactor renders * Some performance optimization * Fix comment * Transform reader * Fix csv test * Fix test * Fix tests * Improve optimaziation * Fix test * Fix test * Detect file encoding with reader * Improve optimaziation * reduce memory usage * improve code * fix build * Fix test * Fix for go1.15 * Fix render * Fix comment * Fix lint * Fix test * Don't use NormalEOF when unnecessary * revert change on util.go * Apply suggestions from code review Co-authored-by: zeripath * rename function * Take NormalEOF back Co-authored-by: zeripath --- modules/markup/csv/csv.go | 123 ++++++++++++++++++++++++++++------------- modules/markup/csv/csv_test.go | 11 +++- 2 files changed, 92 insertions(+), 42 deletions(-) (limited to 'modules/markup/csv') diff --git a/modules/markup/csv/csv.go b/modules/markup/csv/csv.go index 68c89166b5..6572b0ee1e 100644 --- a/modules/markup/csv/csv.go +++ b/modules/markup/csv/csv.go @@ -5,9 +5,11 @@ package markup import ( + "bufio" "bytes" "html" "io" + "io/ioutil" "strconv" "code.gitea.io/gitea/modules/csv" @@ -16,55 +18,89 @@ import ( ) func init() { - markup.RegisterParser(Parser{}) + markup.RegisterRenderer(Renderer{}) } -// Parser implements markup.Parser for csv files -type Parser struct { +// Renderer implements markup.Renderer for csv files +type Renderer struct { } -// Name implements markup.Parser -func (Parser) Name() string { +// Name implements markup.Renderer +func (Renderer) Name() string { return "csv" } -// NeedPostProcess implements markup.Parser -func (Parser) NeedPostProcess() bool { return false } +// NeedPostProcess implements markup.Renderer +func (Renderer) NeedPostProcess() bool { return false } -// Extensions implements markup.Parser -func (Parser) Extensions() []string { +// Extensions implements markup.Renderer +func (Renderer) Extensions() []string { return []string{".csv", ".tsv"} } -// Render implements markup.Parser -func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte { - var tmpBlock bytes.Buffer - - if setting.UI.CSV.MaxFileSize != 0 && setting.UI.CSV.MaxFileSize < int64(len(rawBytes)) { - tmpBlock.WriteString("
")
-		tmpBlock.WriteString(html.EscapeString(string(rawBytes)))
-		tmpBlock.WriteString("
") - return tmpBlock.Bytes() +func writeField(w io.Writer, element, class, field string) error { + if _, err := io.WriteString(w, "<"); err != nil { + return err + } + if _, err := io.WriteString(w, element); err != nil { + return err + } + if len(class) > 0 { + if _, err := io.WriteString(w, " class=\""); err != nil { + return err + } + if _, err := io.WriteString(w, class); err != nil { + return err + } + if _, err := io.WriteString(w, "\""); err != nil { + return err + } + } + if _, err := io.WriteString(w, ">"); err != nil { + return err + } + if _, err := io.WriteString(w, html.EscapeString(field)); err != nil { + return err } + if _, err := io.WriteString(w, "") + return err +} - rd := csv.CreateReaderAndGuessDelimiter(rawBytes) +// Render implements markup.Renderer +func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { + var tmpBlock = bufio.NewWriter(output) - writeField := func(element, class, field string) { - tmpBlock.WriteString("<") - tmpBlock.WriteString(element) - if len(class) > 0 { - tmpBlock.WriteString(" class=\"") - tmpBlock.WriteString(class) - tmpBlock.WriteString("\"") + // FIXME: don't read all to memory + rawBytes, err := ioutil.ReadAll(input) + if err != nil { + return err + } + + if setting.UI.CSV.MaxFileSize != 0 && setting.UI.CSV.MaxFileSize < int64(len(rawBytes)) { + if _, err := tmpBlock.WriteString("
"); err != nil {
+			return err
 		}
-		tmpBlock.WriteString(">")
-		tmpBlock.WriteString(html.EscapeString(field))
-		tmpBlock.WriteString("")
+		if _, err := tmpBlock.WriteString(html.EscapeString(string(rawBytes))); err != nil {
+			return err
+		}
+		_, err = tmpBlock.WriteString("
") + return err + } + + rd, err := csv.CreateReaderAndGuessDelimiter(bytes.NewReader(rawBytes)) + if err != nil { + return err } - tmpBlock.WriteString(``) + if _, err := tmpBlock.WriteString(`
`); err != nil { + return err + } row := 1 for { fields, err := rd.Read() @@ -74,20 +110,29 @@ func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, if err != nil { continue } - tmpBlock.WriteString("") + if _, err := tmpBlock.WriteString(""); err != nil { + return err + } element := "td" if row == 1 { element = "th" } - writeField(element, "line-num", strconv.Itoa(row)) + if err := writeField(tmpBlock, element, "line-num", strconv.Itoa(row)); err != nil { + return err + } for _, field := range fields { - writeField(element, "", field) + if err := writeField(tmpBlock, element, "", field); err != nil { + return err + } + } + if _, err := tmpBlock.WriteString(""); err != nil { + return err } - tmpBlock.WriteString("") row++ } - tmpBlock.WriteString("
") - - return tmpBlock.Bytes() + if _, err = tmpBlock.WriteString(""); err != nil { + return err + } + return tmpBlock.Flush() } diff --git a/modules/markup/csv/csv_test.go b/modules/markup/csv/csv_test.go index 5438ebdf5c..613762f86c 100644 --- a/modules/markup/csv/csv_test.go +++ b/modules/markup/csv/csv_test.go @@ -5,13 +5,16 @@ package markup import ( + "strings" "testing" + "code.gitea.io/gitea/modules/markup" + "github.com/stretchr/testify/assert" ) func TestRenderCSV(t *testing.T) { - var parser Parser + var render Renderer var kases = map[string]string{ "a": "
1a
", "1,2": "
112
", @@ -20,7 +23,9 @@ func TestRenderCSV(t *testing.T) { } for k, v := range kases { - res := parser.Render([]byte(k), "", nil, false) - assert.EqualValues(t, v, string(res)) + var buf strings.Builder + err := render.Render(&markup.RenderContext{}, strings.NewReader(k), &buf) + assert.NoError(t, err) + assert.EqualValues(t, v, buf.String()) } } -- cgit v1.2.3