diff options
Diffstat (limited to 'modules')
| -rw-r--r-- | modules/csv/csv.go | 20 |
1 files changed, 6 insertions, 14 deletions
diff --git a/modules/csv/csv.go b/modules/csv/csv.go index 2e2611aa81..c239c7f647 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -27,6 +27,7 @@ func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader { } // CreateReaderAndGuessDelimiter tries to guess the field delimiter from the content and creates a csv.Reader. +// Reads at most 10k bytes. func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) { var data = make([]byte, 1e4) size, err := util.ReadAtMost(rd, data) @@ -34,25 +35,16 @@ func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) { return nil, err } - delimiter := guessDelimiter(data[:size]) - - var newInput io.Reader - if size < 1e4 { - newInput = bytes.NewReader(data[:size]) - } else { - newInput = io.MultiReader(bytes.NewReader(data), rd) - } - - return CreateReader(newInput, delimiter), nil + return CreateReader( + io.MultiReader(bytes.NewReader(data[:size]), rd), + guessDelimiter(data[:size]), + ), nil } // guessDelimiter scores the input CSV data against delimiters, and returns the best match. -// Reads at most 10k bytes & 10 lines. func guessDelimiter(data []byte) rune { maxLines := 10 - maxBytes := util.Min(len(data), 1e4) - text := string(data[:maxBytes]) - text = quoteRegexp.ReplaceAllLiteralString(text, "") + text := quoteRegexp.ReplaceAllLiteralString(string(data), "") lines := strings.SplitN(text, "\n", maxLines+1) lines = lines[:util.Min(maxLines, len(lines))] |