From d33485619a9a18867c464c8b3b62dfcb17888647 Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Sun, 15 Sep 2019 13:54:17 -0300 Subject: [PATCH 1/3] Prevent compiler environment from making the tests fail --- modules/charset/charset.go | 2 +- modules/charset/charset_test.go | 119 +++++++++++++++++++++++--------- 2 files changed, 86 insertions(+), 35 deletions(-) diff --git a/modules/charset/charset.go b/modules/charset/charset.go index f43fe387edfde..d3c5ac0b4257c 100644 --- a/modules/charset/charset.go +++ b/modules/charset/charset.go @@ -35,7 +35,7 @@ func ToUTF8WithErr(content []byte) (string, error) { } // If there is an error, we concatenate the nicely decoded part and the - // original left over. This way we won't lose data. + // original left over. This way we won't lose much data. result, n, err := transform.Bytes(encoding.NewDecoder(), content) if err != nil { result = append(result, content[n:]...) diff --git a/modules/charset/charset_test.go b/modules/charset/charset_test.go index fde42732e80ec..5847c4c6a14ff 100644 --- a/modules/charset/charset_test.go +++ b/modules/charset/charset_test.go @@ -24,41 +24,55 @@ func TestToUTF8WithErr(t *testing.T) { var res string var err error + // Note: golang compiler seems so behave differently depending on the current + // locale, so some conversions might behave differently. For that reason, we don't + // depend on particular conversions but in expected behaviors. + res, err = ToUTF8WithErr([]byte{0x41, 0x42, 0x43}) - assert.Equal(t, "ABC", res) assert.NoError(t, err) + assert.Equal(t, "ABC", res) + // "áéíóú" res, err = ToUTF8WithErr([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}) - assert.Equal(t, "áéíóú", res) assert.NoError(t, err) + assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res)) - res, err = ToUTF8WithErr([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}) - assert.Equal(t, "áéíóú", res) + // "áéíóú" + res, err = ToUTF8WithErr([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, + 0xc3, 0xba}) assert.NoError(t, err) + assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res)) - // This test FAILS - res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73}) - assert.Equal(t, "Hola, así cómo ños", res) + res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, + 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e}) assert.NoError(t, err) + stringMustStartWith(t, "Hola,", res) + stringMustEndWith(t, "AAA.", res) - res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73}) - // Do not fail for differences in invalid cases, as the library might change the conversion criteria for those - assert.Regexp(t, "^Hola, así cómo", res) + res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, + 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e}) assert.NoError(t, err) + stringMustStartWith(t, "Hola,", res) + stringMustEndWith(t, "AAA.", res) - res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73}) - // Do not fail for differences in invalid cases, as the library might change the conversion criteria for those - assert.Regexp(t, "^Hola, así cómo", res) + res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, + 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e}) assert.NoError(t, err) + stringMustStartWith(t, "Hola,", res) + stringMustEndWith(t, "AAA.", res) // Japanese (Shift-JIS) - res, err = ToUTF8WithErr([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42}) - assert.Equal(t, "日属秘ぞしちゅ。", res) + // 日属秘ぞしちゅ。 + res, err = ToUTF8WithErr([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, + 0xBF, 0x82, 0xE3, 0x81, 0x42}) assert.NoError(t, err) + assert.Equal(t, []byte{0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3, + 0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82}, + []byte(res)) res, err = ToUTF8WithErr([]byte{0x00, 0x00, 0x00, 0x00}) - assert.Equal(t, "\x00\x00\x00\x00", res) assert.NoError(t, err) + assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, []byte(res)) } func TestToUTF8WithFallback(t *testing.T) { @@ -75,8 +89,10 @@ func TestToUTF8WithFallback(t *testing.T) { assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res) // "Hola, así cómo ños" - res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73}) - assert.Equal(t, []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, 0xC3, 0xB3, 0x6D, 0x6F, 0x20, 0xC3, 0xB1, 0x6F, 0x73}, res) + res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, + 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73}) + assert.Equal(t, []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, + 0xC3, 0xB3, 0x6D, 0x6F, 0x20, 0xC3, 0xB1, 0x6F, 0x73}, res) // "Hola, así cómo " minmatch := []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, 0xC3, 0xB3, 0x6D, 0x6F, 0x20} @@ -100,22 +116,37 @@ func TestToUTF8WithFallback(t *testing.T) { } func TestToUTF8(t *testing.T) { - res := ToUTF8("ABC") - assert.Equal(t, "ABC", res) - - res = ToUTF8("áéíóú") - assert.Equal(t, "áéíóú", res) + var res string - // With utf-8 BOM - res = ToUTF8("\ufeffáéíóú") - assert.Equal(t, "áéíóú", res) + // Note: golang compiler seems so behave differently depending on the current + // locale, so some conversions might behave differently. For that reason, we don't + // depend on particular conversions but in expected behaviors. - res = ToUTF8("Hola, así cómo ños") - assert.Equal(t, "Hola, así cómo ños", res) + res = ToUTF8(string([]byte{0x41, 0x42, 0x43})) + assert.Equal(t, "ABC", res) - res = ToUTF8("Hola, así cómo \x07ños") - // Do not fail for differences in invalid cases, as the library might change the conversion criteria for those - assert.Regexp(t, "^Hola, así cómo", res) + // "áéíóú" + res = ToUTF8(string([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})) + assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res)) + + // BOM + "áéíóú" + res = ToUTF8(string([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, + 0xc3, 0xba})) + assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res)) + + // Latin1 + // Hola, así cómo ños + res = ToUTF8(string([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, + 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73})) + assert.Equal(t, []byte{0x48, 0x6f, 0x6c, 0x61, 0x2c, 0x20, 0x61, 0x73, 0xc3, 0xad, 0x20, 0x63, + 0xc3, 0xb3, 0x6d, 0x6f, 0x20, 0xc3, 0xb1, 0x6f, 0x73}, []byte(res)) + + // Latin1 + // Hola, así cómo \x07ños + res = ToUTF8(string([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, + 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73})) + // Hola, + bytesMustStartWith(t, []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C}, []byte(res)) // This test FAILS // res = ToUTF8("Hola, así cómo \x81ños") @@ -123,11 +154,15 @@ func TestToUTF8(t *testing.T) { // assert.Regexp(t, "^Hola, así cómo", res) // Japanese (Shift-JIS) - res = ToUTF8("\x93\xFA\x91\xAE\x94\xE9\x82\xBC\x82\xB5\x82\xBF\x82\xE3\x81\x42") - assert.Equal(t, "日属秘ぞしちゅ。", res) + // 日属秘ぞしちゅ。 + res = ToUTF8(string([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, + 0xBF, 0x82, 0xE3, 0x81, 0x42})) + assert.Equal(t, []byte{0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3, + 0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82}, + []byte(res)) res = ToUTF8("\x00\x00\x00\x00") - assert.Equal(t, "\x00\x00\x00\x00", res) + assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, []byte(res)) } func TestToUTF8DropErrors(t *testing.T) { @@ -203,3 +238,19 @@ func TestDetectEncoding(t *testing.T) { _, err = DetectEncoding(b) assert.Error(t, err) } + +func stringMustStartWith(t *testing.T, expected string, value string) { + assert.Equal(t, expected, string(value[:len(expected)])) +} + +func stringMustEndWith(t *testing.T, expected string, value string) { + assert.Equal(t, expected, string(value[len(value)-len(expected):])) +} + +func bytesMustStartWith(t *testing.T, expected []byte, value []byte) { + assert.Equal(t, expected, value[:len(expected)]) +} + +func bytesMustEndWith(t *testing.T, expected []byte, value []byte) { + assert.Equal(t, expected, value[len(value)-len(expected):]) +} From 9229e50ba27fbd493a4f594d98edab4519559d47 Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Sun, 15 Sep 2019 14:26:32 -0300 Subject: [PATCH 2/3] Remove unused function --- modules/charset/charset_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modules/charset/charset_test.go b/modules/charset/charset_test.go index 5847c4c6a14ff..776745b96f649 100644 --- a/modules/charset/charset_test.go +++ b/modules/charset/charset_test.go @@ -250,7 +250,3 @@ func stringMustEndWith(t *testing.T, expected string, value string) { func bytesMustStartWith(t *testing.T, expected []byte, value []byte) { assert.Equal(t, expected, value[:len(expected)]) } - -func bytesMustEndWith(t *testing.T, expected []byte, value []byte) { - assert.Equal(t, expected, value[len(value)-len(expected):]) -} From 602c53b0f9a34397ca657f8349b5c917078d3897 Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Sun, 15 Sep 2019 15:29:03 -0300 Subject: [PATCH 3/3] Pass lint --- modules/charset/charset_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/charset/charset_test.go b/modules/charset/charset_test.go index 776745b96f649..3c77f127894df 100644 --- a/modules/charset/charset_test.go +++ b/modules/charset/charset_test.go @@ -116,13 +116,12 @@ func TestToUTF8WithFallback(t *testing.T) { } func TestToUTF8(t *testing.T) { - var res string // Note: golang compiler seems so behave differently depending on the current // locale, so some conversions might behave differently. For that reason, we don't // depend on particular conversions but in expected behaviors. - res = ToUTF8(string([]byte{0x41, 0x42, 0x43})) + res := ToUTF8(string([]byte{0x41, 0x42, 0x43})) assert.Equal(t, "ABC", res) // "áéíóú"