Clarify Gitea/Crowdin locale behaviors, add tests for LocaleStore, fix some strings with semicolons (#23819)

Follow #23633 and #23240

Close #23814

Now we almost have a complete test set for Gitea's LocalStore.

This PR is still a quick fix for the legacy locale system (see the
TODOs), to resolve the problems fundamentally, it needs more work in the
future.

1 files changed, 54 insertions, 0 deletions

diff --git a/modules/translation/i18n/i18n_test.go b/modules/translation/i18n/i18n_test.go
index 76a91522ba..085d03811f 100644
--- a/modules/translation/i18n/i18n_test.go
+++ b/modules/translation/i18n/i18n_test.go
@@ -4,6 +4,7 @@
 package i18n
 
 import (
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -75,3 +76,56 @@ c=22
 	assert.Equal(t, "21", ls.Tr("lang1", "b"))
 	assert.Equal(t, "22", ls.Tr("lang1", "c"))
 }
+
+func TestLocaleStoreQuirks(t *testing.T) {
+	const nl = "\n"
+	q := func(q1, s string, q2 ...string) string {
+		return q1 + s + strings.Join(q2, "")
+	}
+	testDataList := []struct {
+		in   string
+		out  string
+		hint string
+	}{
+		{` xx`, `xx`, "simple, no quote"},
+		{`" xx"`, ` xx`, "simple, double-quote"},
+		{`' xx'`, ` xx`, "simple, single-quote"},
+		{"` xx`", ` xx`, "simple, back-quote"},
+
+		{`x\"y`, `x\"y`, "no unescape, simple"},
+		{q(`"`, `x\"y`, `"`), `"x\"y"`, "unescape, double-quote"},
+		{q(`'`, `x\"y`, `'`), `x\"y`, "no unescape, single-quote"},
+		{q("`", `x\"y`, "`"), `x\"y`, "no unescape, back-quote"},
+
+		{q(`"`, `x\"y`) + nl + "b=", `"x\"y`, "half open, double-quote"},
+		{q(`'`, `x\"y`) + nl + "b=", `'x\"y`, "half open, single-quote"},
+		{q("`", `x\"y`) + nl + "b=`", `x\"y` + nl + "b=", "half open, back-quote, multi-line"},
+
+		{`x ; y`, `x ; y`, "inline comment (;)"},
+		{`x # y`, `x # y`, "inline comment (#)"},
+		{`x \; y`, `x ; y`, `inline comment (\;)`},
+		{`x \# y`, `x # y`, `inline comment (\#)`},
+	}
+
+	for _, testData := range testDataList {
+		ls := NewLocaleStore()
+		err := ls.AddLocaleByIni("lang1", "Lang1", []byte("a="+testData.in), nil)
+		assert.NoError(t, err, testData.hint)
+		assert.Equal(t, testData.out, ls.Tr("lang1", "a"), testData.hint)
+		assert.NoError(t, ls.Close())
+	}
+
+	// TODO: Crowdin needs the strings to be quoted correctly and doesn't like incomplete quotes
+	//       and Crowdin always outputs quoted strings if there are quotes in the strings.
+	//       So, Gitea's `key="quoted" unquoted` content shouldn't be used on Crowdin directly,
+	//       it should be converted to `key="\"quoted\" unquoted"` first.
+	// TODO: We can not use UnescapeValueDoubleQuotes=true, because there are a lot of back-quotes in en-US.ini,
+	//       then Crowdin will output:
+	//       > key = "`x \" y`"
+	//       Then Gitea will read a string with back-quotes, which is incorrect.
+	// TODO: Crowdin might generate multi-line strings, quoted by double-quote, it's not supported by LocaleStore
+	//       LocaleStore uses back-quote for multi-line strings, it's not supported by Crowdin.
+	// TODO: Crowdin doesn't support back-quote as string quoter, it mainly uses double-quote
+	//       so, the following line will be parsed as: value="`first", comment="second`" on Crowdin
+	//       > a = `first; second`
+}