diff options
Diffstat (limited to 'modules/csv')
-rw-r--r-- | modules/csv/csv.go | 26 | ||||
-rw-r--r-- | modules/csv/csv_test.go | 7 |
2 files changed, 26 insertions, 7 deletions
diff --git a/modules/csv/csv.go b/modules/csv/csv.go index 1aa78fdeec..bf433f77d2 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -7,7 +7,9 @@ package csv import ( "bytes" "encoding/csv" + stdcsv "encoding/csv" "errors" + "io" "regexp" "strings" @@ -18,17 +20,31 @@ import ( var quoteRegexp = regexp.MustCompile(`["'][\s\S]+?["']`) // CreateReader creates a csv.Reader with the given delimiter. -func CreateReader(rawBytes []byte, delimiter rune) *csv.Reader { - rd := csv.NewReader(bytes.NewReader(rawBytes)) +func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader { + rd := stdcsv.NewReader(input) rd.Comma = delimiter rd.TrimLeadingSpace = true return rd } // CreateReaderAndGuessDelimiter tries to guess the field delimiter from the content and creates a csv.Reader. -func CreateReaderAndGuessDelimiter(rawBytes []byte) *csv.Reader { - delimiter := guessDelimiter(rawBytes) - return CreateReader(rawBytes, delimiter) +func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) { + var data = make([]byte, 1e4) + size, err := rd.Read(data) + if err != nil { + return nil, err + } + + delimiter := guessDelimiter(data[:size]) + + var newInput io.Reader + if size < 1e4 { + newInput = bytes.NewReader(data[:size]) + } else { + newInput = io.MultiReader(bytes.NewReader(data), rd) + } + + return CreateReader(newInput, delimiter), nil } // guessDelimiter scores the input CSV data against delimiters, and returns the best match. diff --git a/modules/csv/csv_test.go b/modules/csv/csv_test.go index 3a7584e21d..3cc09c40aa 100644 --- a/modules/csv/csv_test.go +++ b/modules/csv/csv_test.go @@ -5,20 +5,23 @@ package csv import ( + "bytes" + "strings" "testing" "github.com/stretchr/testify/assert" ) func TestCreateReader(t *testing.T) { - rd := CreateReader([]byte{}, ',') + rd := CreateReader(bytes.NewReader([]byte{}), ',') assert.Equal(t, ',', rd.Comma) } func TestCreateReaderAndGuessDelimiter(t *testing.T) { input := "a;b;c\n1;2;3\n4;5;6" - rd := CreateReaderAndGuessDelimiter([]byte(input)) + rd, err := CreateReaderAndGuessDelimiter(strings.NewReader(input)) + assert.NoError(t, err) assert.Equal(t, ';', rd.Comma) } |