Fix utf8 tests (#8192)

* Prevent compiler environment from making the tests fail

* Remove unused function

* Pass lint

2 files changed, 81 insertions, 35 deletions

diff --git a/modules/charset/charset.go b/modules/charset/charset.go
index f43fe387ed..d3c5ac0b42 100644
--- a/modules/charset/charset.go
+++ b/modules/charset/charset.go
@@ -35,7 +35,7 @@ func ToUTF8WithErr(content []byte) (string, error) {
 	}
 
 	// If there is an error, we concatenate the nicely decoded part and the
-	// original left over. This way we won't lose data.
+	// original left over. This way we won't lose much data.
 	result, n, err := transform.Bytes(encoding.NewDecoder(), content)
 	if err != nil {
 		result = append(result, content[n:]...)
diff --git a/modules/charset/charset_test.go b/modules/charset/charset_test.go
index fde42732e8..3c77f12789 100644
--- a/modules/charset/charset_test.go
+++ b/modules/charset/charset_test.go
@@ -24,41 +24,55 @@ func TestToUTF8WithErr(t *testing.T) {
 	var res string
 	var err error
 
+	// Note: golang compiler seems so behave differently depending on the current
+	// locale, so some conversions might behave differently. For that reason, we don't
+	// depend on particular conversions but in expected behaviors.
+
 	res, err = ToUTF8WithErr([]byte{0x41, 0x42, 0x43})
-	assert.Equal(t, "ABC", res)
 	assert.NoError(t, err)
+	assert.Equal(t, "ABC", res)
 
+	// "áéíóú"
 	res, err = ToUTF8WithErr([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
-	assert.Equal(t, "áéíóú", res)
 	assert.NoError(t, err)
+	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
 
-	res, err = ToUTF8WithErr([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
-	assert.Equal(t, "áéíóú", res)
+	// "áéíóú"
+	res, err = ToUTF8WithErr([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3,
+		0xc3, 0xba})
 	assert.NoError(t, err)
+	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
 
-	// This test FAILS
-	res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73})
-	assert.Equal(t, "Hola, así cómo ños", res)
+	res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
+		0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e})
 	assert.NoError(t, err)
+	stringMustStartWith(t, "Hola,", res)
+	stringMustEndWith(t, "AAA.", res)
 
-	res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73})
-	// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
-	assert.Regexp(t, "^Hola, así cómo", res)
+	res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
+		0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e})
 	assert.NoError(t, err)
+	stringMustStartWith(t, "Hola,", res)
+	stringMustEndWith(t, "AAA.", res)
 
-	res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73})
-	// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
-	assert.Regexp(t, "^Hola, así cómo", res)
+	res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
+		0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e})
 	assert.NoError(t, err)
+	stringMustStartWith(t, "Hola,", res)
+	stringMustEndWith(t, "AAA.", res)
 
 	// Japanese (Shift-JIS)
-	res, err = ToUTF8WithErr([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42})
-	assert.Equal(t, "日属秘ぞしちゅ。", res)
+	// 日属秘ぞしちゅ。
+	res, err = ToUTF8WithErr([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82,
+		0xBF, 0x82, 0xE3, 0x81, 0x42})
 	assert.NoError(t, err)
+	assert.Equal(t, []byte{0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
+		0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82},
+		[]byte(res))
 
 	res, err = ToUTF8WithErr([]byte{0x00, 0x00, 0x00, 0x00})
-	assert.Equal(t, "\x00\x00\x00\x00", res)
 	assert.NoError(t, err)
+	assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, []byte(res))
 }
 
 func TestToUTF8WithFallback(t *testing.T) {
@@ -75,8 +89,10 @@ func TestToUTF8WithFallback(t *testing.T) {
 	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
 
 	// "Hola, así cómo ños"
-	res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73})
-	assert.Equal(t, []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, 0xC3, 0xB3, 0x6D, 0x6F, 0x20, 0xC3, 0xB1, 0x6F, 0x73}, res)
+	res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
+		0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73})
+	assert.Equal(t, []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63,
+		0xC3, 0xB3, 0x6D, 0x6F, 0x20, 0xC3, 0xB1, 0x6F, 0x73}, res)
 
 	// "Hola, así cómo "
 	minmatch := []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, 0xC3, 0xB3, 0x6D, 0x6F, 0x20}
@@ -100,22 +116,36 @@ func TestToUTF8WithFallback(t *testing.T) {
 }
 
 func TestToUTF8(t *testing.T) {
-	res := ToUTF8("ABC")
-	assert.Equal(t, "ABC", res)
 
-	res = ToUTF8("áéíóú")
-	assert.Equal(t, "áéíóú", res)
+	// Note: golang compiler seems so behave differently depending on the current
+	// locale, so some conversions might behave differently. For that reason, we don't
+	// depend on particular conversions but in expected behaviors.
 
-	// With utf-8 BOM
-	res = ToUTF8("\ufeffáéíóú")
-	assert.Equal(t, "áéíóú", res)
-
-	res = ToUTF8("Hola, así cómo ños")
-	assert.Equal(t, "Hola, así cómo ños", res)
+	res := ToUTF8(string([]byte{0x41, 0x42, 0x43}))
+	assert.Equal(t, "ABC", res)
 
-	res = ToUTF8("Hola, así cómo \x07ños")
-	// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
-	assert.Regexp(t, "^Hola, así cómo", res)
+	// "áéíóú"
+	res = ToUTF8(string([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}))
+	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
+
+	// BOM + "áéíóú"
+	res = ToUTF8(string([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3,
+		0xc3, 0xba}))
+	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
+
+	// Latin1
+	// Hola, así cómo ños
+	res = ToUTF8(string([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
+		0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73}))
+	assert.Equal(t, []byte{0x48, 0x6f, 0x6c, 0x61, 0x2c, 0x20, 0x61, 0x73, 0xc3, 0xad, 0x20, 0x63,
+		0xc3, 0xb3, 0x6d, 0x6f, 0x20, 0xc3, 0xb1, 0x6f, 0x73}, []byte(res))
+
+	// Latin1
+	// Hola, así cómo \x07ños
+	res = ToUTF8(string([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
+		0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73}))
+	// Hola,
+	bytesMustStartWith(t, []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C}, []byte(res))
 
 	// This test FAILS
 	// res = ToUTF8("Hola, así cómo \x81ños")
@@ -123,11 +153,15 @@ func TestToUTF8(t *testing.T) {
 	// assert.Regexp(t, "^Hola, así cómo", res)
 
 	// Japanese (Shift-JIS)
-	res = ToUTF8("\x93\xFA\x91\xAE\x94\xE9\x82\xBC\x82\xB5\x82\xBF\x82\xE3\x81\x42")
-	assert.Equal(t, "日属秘ぞしちゅ。", res)
+	// 日属秘ぞしちゅ。
+	res = ToUTF8(string([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82,
+		0xBF, 0x82, 0xE3, 0x81, 0x42}))
+	assert.Equal(t, []byte{0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
+		0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82},
+		[]byte(res))
 
 	res = ToUTF8("\x00\x00\x00\x00")
-	assert.Equal(t, "\x00\x00\x00\x00", res)
+	assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, []byte(res))
 }
 
 func TestToUTF8DropErrors(t *testing.T) {
@@ -203,3 +237,15 @@ func TestDetectEncoding(t *testing.T) {
 	_, err = DetectEncoding(b)
 	assert.Error(t, err)
 }
+
+func stringMustStartWith(t *testing.T, expected string, value string) {
+	assert.Equal(t, expected, string(value[:len(expected)]))
+}
+
+func stringMustEndWith(t *testing.T, expected string, value string) {
+	assert.Equal(t, expected, string(value[len(value)-len(expected):]))
+}
+
+func bytesMustStartWith(t *testing.T, expected []byte, value []byte) {
+	assert.Equal(t, expected, value[:len(expected)])
+}