diff options
author | Lunny Xiao <xiaolunwen@gmail.com> | 2021-04-20 06:25:08 +0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-04-19 18:25:08 -0400 |
commit | 9d99f6ab19ac3f97af3ca126720e9075c127a652 (patch) | |
tree | b817b4582a871f83b91ad7977fe772fc3501c1e8 /modules/csv | |
parent | c9cc6698d2172625854cd063301e63602204a2a1 (diff) | |
download | gitea-9d99f6ab19ac3f97af3ca126720e9075c127a652.tar.gz gitea-9d99f6ab19ac3f97af3ca126720e9075c127a652.zip |
Refactor renders (#15175)
* Refactor renders
* Some performance optimization
* Fix comment
* Transform reader
* Fix csv test
* Fix test
* Fix tests
* Improve optimaziation
* Fix test
* Fix test
* Detect file encoding with reader
* Improve optimaziation
* reduce memory usage
* improve code
* fix build
* Fix test
* Fix for go1.15
* Fix render
* Fix comment
* Fix lint
* Fix test
* Don't use NormalEOF when unnecessary
* revert change on util.go
* Apply suggestions from code review
Co-authored-by: zeripath <art27@cantab.net>
* rename function
* Take NormalEOF back
Co-authored-by: zeripath <art27@cantab.net>
Diffstat (limited to 'modules/csv')
-rw-r--r-- | modules/csv/csv.go | 26 | ||||
-rw-r--r-- | modules/csv/csv_test.go | 7 |
2 files changed, 26 insertions, 7 deletions
diff --git a/modules/csv/csv.go b/modules/csv/csv.go index 1aa78fdeec..bf433f77d2 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -7,7 +7,9 @@ package csv import ( "bytes" "encoding/csv" + stdcsv "encoding/csv" "errors" + "io" "regexp" "strings" @@ -18,17 +20,31 @@ import ( var quoteRegexp = regexp.MustCompile(`["'][\s\S]+?["']`) // CreateReader creates a csv.Reader with the given delimiter. -func CreateReader(rawBytes []byte, delimiter rune) *csv.Reader { - rd := csv.NewReader(bytes.NewReader(rawBytes)) +func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader { + rd := stdcsv.NewReader(input) rd.Comma = delimiter rd.TrimLeadingSpace = true return rd } // CreateReaderAndGuessDelimiter tries to guess the field delimiter from the content and creates a csv.Reader. -func CreateReaderAndGuessDelimiter(rawBytes []byte) *csv.Reader { - delimiter := guessDelimiter(rawBytes) - return CreateReader(rawBytes, delimiter) +func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) { + var data = make([]byte, 1e4) + size, err := rd.Read(data) + if err != nil { + return nil, err + } + + delimiter := guessDelimiter(data[:size]) + + var newInput io.Reader + if size < 1e4 { + newInput = bytes.NewReader(data[:size]) + } else { + newInput = io.MultiReader(bytes.NewReader(data), rd) + } + + return CreateReader(newInput, delimiter), nil } // guessDelimiter scores the input CSV data against delimiters, and returns the best match. diff --git a/modules/csv/csv_test.go b/modules/csv/csv_test.go index 3a7584e21d..3cc09c40aa 100644 --- a/modules/csv/csv_test.go +++ b/modules/csv/csv_test.go @@ -5,20 +5,23 @@ package csv import ( + "bytes" + "strings" "testing" "github.com/stretchr/testify/assert" ) func TestCreateReader(t *testing.T) { - rd := CreateReader([]byte{}, ',') + rd := CreateReader(bytes.NewReader([]byte{}), ',') assert.Equal(t, ',', rd.Comma) } func TestCreateReaderAndGuessDelimiter(t *testing.T) { input := "a;b;c\n1;2;3\n4;5;6" - rd := CreateReaderAndGuessDelimiter([]byte(input)) + rd, err := CreateReaderAndGuessDelimiter(strings.NewReader(input)) + assert.NoError(t, err) assert.Equal(t, ';', rd.Comma) } |