From ec3137baf34471af66654490e4692626a17911c8 Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Tue, 26 Oct 2021 11:15:37 -0400 Subject: [PATCH 1/4] Fixes #16559 - Do not trim leading spaces for tab delimited --- modules/csv/csv.go | 6 ++++- modules/csv/csv_test.go | 51 +++++++++++++++++++++++++++++++++++------ 2 files changed, 49 insertions(+), 8 deletions(-) diff --git a/modules/csv/csv.go b/modules/csv/csv.go index c239c7f647ace..cba23ec8d9df7 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -22,7 +22,11 @@ var quoteRegexp = regexp.MustCompile(`["'][\s\S]+?["']`) func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader { rd := stdcsv.NewReader(input) rd.Comma = delimiter - rd.TrimLeadingSpace = true + if delimiter != '\t' && delimiter != ' ' { + // TrimLeadingSpace can't be true when delimiter is a tab or a space as the value for a column might be empty, + // thus would change `\t\t` to just `\t` or ` ` (two spaces) to just ` ` (single space) + rd.TrimLeadingSpace = true + } return rd } diff --git a/modules/csv/csv_test.go b/modules/csv/csv_test.go index 3cc09c40aa5b3..f1d857fd3e12e 100644 --- a/modules/csv/csv_test.go +++ b/modules/csv/csv_test.go @@ -18,15 +18,52 @@ func TestCreateReader(t *testing.T) { } func TestCreateReaderAndGuessDelimiter(t *testing.T) { - input := "a;b;c\n1;2;3\n4;5;6" + var csvToRowsMap = map[string][][]string{ + `col1 col2 col3 +a b c + e f +g h i +j l +m n +p q r + u +v w x +y + `: {{"col1", "col2", "col3"}, + {"a", "b", "c"}, + {"", "e", "f"}, + {"g", "h", "i"}, + {"j", "", "l"}, + {"m", "n", ""}, + {"p", "q", "r"}, + {"", "", "u"}, + {"v", "w", "x"}, + {"y", "", ""}, + {"", "", ""}}, + ` col1,col2,col3 + a, b, c +d,e,f + ,h, i +j, , + , , `: {{"col1", "col2", "col3"}, + {"a", "b", "c"}, + {"d", "e", "f"}, + {"", "h", "i"}, + {"j", "", ""}, + {"", "", ""}}, + } - rd, err := CreateReaderAndGuessDelimiter(strings.NewReader(input)) - assert.NoError(t, err) - assert.Equal(t, ';', rd.Comma) + for csv, expectedRows := range csvToRowsMap { + rd, err := CreateReaderAndGuessDelimiter(strings.NewReader(csv)) + assert.NoError(t, err) + rows, err := rd.ReadAll() + assert.NoError(t, err) + assert.EqualValues(t, rows, expectedRows) + } } func TestGuessDelimiter(t *testing.T) { - var kases = map[string]rune{ + var csvToDelimiterMap = map[string]rune{ "a": ',', "1,2": ',', "1;2": ';', @@ -37,7 +74,7 @@ func TestGuessDelimiter(t *testing.T) { "
": ',', } - for k, v := range kases { - assert.EqualValues(t, guessDelimiter([]byte(k)), v) + for csv, expectedDelimiter := range csvToDelimiterMap { + assert.EqualValues(t, guessDelimiter([]byte(csv)), expectedDelimiter) } } From a4f0782acf345413f4fd7e0305ea7d1243788662 Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Tue, 26 Oct 2021 11:31:22 -0400 Subject: [PATCH 2/4] Adds back semicolon delimited test --- modules/csv/csv_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/csv/csv_test.go b/modules/csv/csv_test.go index f1d857fd3e12e..ae50f114ffb21 100644 --- a/modules/csv/csv_test.go +++ b/modules/csv/csv_test.go @@ -19,6 +19,9 @@ func TestCreateReader(t *testing.T) { func TestCreateReaderAndGuessDelimiter(t *testing.T) { var csvToRowsMap = map[string][][]string{ + `a;b;c +1;2;3 +4;5;6`: {{"a", "b", "c"}, {"1", "2", "3"}, {"4", "5", "6"}}, `col1 col2 col3 a b c e f From 18802c4b3a8653f4ff3fc77206bf148547e444d3 Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Tue, 26 Oct 2021 11:51:43 -0400 Subject: [PATCH 3/4] Fixes linting --- modules/csv/csv_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/csv/csv_test.go b/modules/csv/csv_test.go index ae50f114ffb21..f71ce0f884f9c 100644 --- a/modules/csv/csv_test.go +++ b/modules/csv/csv_test.go @@ -43,7 +43,7 @@ y {"v", "w", "x"}, {"y", "", ""}, {"", "", ""}}, - ` col1,col2,col3 + `col1,col2,col3 a, b, c d,e,f ,h, i From ef4d82ebb1a29c27d9e6d5ec1bad5475c4883af8 Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Tue, 26 Oct 2021 12:20:01 -0400 Subject: [PATCH 4/4] Adds nolint directive to test because uses strings starting with spaces --- modules/csv/csv_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/csv/csv_test.go b/modules/csv/csv_test.go index f71ce0f884f9c..9b7fa1f4fafc2 100644 --- a/modules/csv/csv_test.go +++ b/modules/csv/csv_test.go @@ -17,6 +17,7 @@ func TestCreateReader(t *testing.T) { assert.Equal(t, ',', rd.Comma) } +//nolint func TestCreateReaderAndGuessDelimiter(t *testing.T) { var csvToRowsMap = map[string][][]string{ `a;b;c @@ -43,7 +44,7 @@ y {"v", "w", "x"}, {"y", "", ""}, {"", "", ""}}, - `col1,col2,col3 + ` col1,col2,col3 a, b, c d,e,f ,h, i