Remove legacy unmaintained packages, refactor to support change default locale (#19308)

Remove two unmaintained vendor packages `i18n` and `paginater`. Changes:
* Rewrite `i18n` package with a more clear fallback mechanism. Fix an unstable `Tr` behavior, add more tests.
* Refactor the legacy `Paginater` to `Paginator`, test cases are kept unchanged.

Trivial enhancement (no breaking for end users):
* Use the first locale in LANGS setting option as the default, add a log to prevent from surprising users.

3 files changed, 212 insertions, 8 deletions

diff --git a/modules/translation/i18n/i18n.go b/modules/translation/i18n/i18n.go
new file mode 100644
index 0000000000..664e457ecf
--- /dev/null
+++ b/modules/translation/i18n/i18n.go
@@ -0,0 +1,143 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package i18n
+
+import (
+	"errors"
+	"fmt"
+	"reflect"
+	"strings"
+
+	"code.gitea.io/gitea/modules/log"
+
+	"gopkg.in/ini.v1"
+)
+
+var (
+	ErrLocaleAlreadyExist = errors.New("lang already exists")
+
+	DefaultLocales = NewLocaleStore()
+)
+
+type locale struct {
+	store    *LocaleStore
+	langName string
+	langDesc string
+	messages *ini.File
+}
+
+type LocaleStore struct {
+	// at the moment, all these fields are readonly after initialization
+	langNames   []string
+	langDescs   []string
+	localeMap   map[string]*locale
+	defaultLang string
+}
+
+func NewLocaleStore() *LocaleStore {
+	return &LocaleStore{localeMap: make(map[string]*locale)}
+}
+
+// AddLocaleByIni adds locale by ini into the store
+func (ls *LocaleStore) AddLocaleByIni(langName, langDesc string, localeFile interface{}, otherLocaleFiles ...interface{}) error {
+	if _, ok := ls.localeMap[langName]; ok {
+		return ErrLocaleAlreadyExist
+	}
+	iniFile, err := ini.LoadSources(ini.LoadOptions{
+		IgnoreInlineComment:         true,
+		UnescapeValueCommentSymbols: true,
+	}, localeFile, otherLocaleFiles...)
+	if err == nil {
+		iniFile.BlockMode = false
+		lc := &locale{store: ls, langName: langName, langDesc: langDesc, messages: iniFile}
+		ls.langNames = append(ls.langNames, lc.langName)
+		ls.langDescs = append(ls.langDescs, lc.langDesc)
+		ls.localeMap[lc.langName] = lc
+	}
+	return err
+}
+
+func (ls *LocaleStore) HasLang(langName string) bool {
+	_, ok := ls.localeMap[langName]
+	return ok
+}
+
+func (ls *LocaleStore) ListLangNameDesc() (names, desc []string) {
+	return ls.langNames, ls.langDescs
+}
+
+// SetDefaultLang sets default language as a fallback
+func (ls *LocaleStore) SetDefaultLang(lang string) {
+	ls.defaultLang = lang
+}
+
+// Tr translates content to target language. fall back to default language.
+func (ls *LocaleStore) Tr(lang, trKey string, trArgs ...interface{}) string {
+	l, ok := ls.localeMap[lang]
+	if !ok {
+		l, ok = ls.localeMap[ls.defaultLang]
+	}
+	if ok {
+		return l.Tr(trKey, trArgs...)
+	}
+	return trKey
+}
+
+// Tr translates content to locale language. fall back to default language.
+func (l *locale) Tr(trKey string, trArgs ...interface{}) string {
+	var section string
+
+	idx := strings.IndexByte(trKey, '.')
+	if idx > 0 {
+		section = trKey[:idx]
+		trKey = trKey[idx+1:]
+	}
+
+	trMsg := trKey
+	if trIni, err := l.messages.Section(section).GetKey(trKey); err == nil {
+		trMsg = trIni.Value()
+	} else if l.store.defaultLang != "" && l.langName != l.store.defaultLang {
+		// try to fall back to default
+		if defaultLocale, ok := l.store.localeMap[l.store.defaultLang]; ok {
+			if trIni, err = defaultLocale.messages.Section(section).GetKey(trKey); err == nil {
+				trMsg = trIni.Value()
+			}
+		}
+	}
+
+	if len(trArgs) > 0 {
+		fmtArgs := make([]interface{}, 0, len(trArgs))
+		for _, arg := range trArgs {
+			val := reflect.ValueOf(arg)
+			if val.Kind() == reflect.Slice {
+				// before, it can accept Tr(lang, key, a, [b, c], d, [e, f]) as Sprintf(msg, a, b, c, d, e, f), it's an unstable behavior
+				// now, we restrict the strange behavior and only support:
+				// 1. Tr(lang, key, [slice-items]) as Sprintf(msg, items...)
+				// 2. Tr(lang, key, args...) as Sprintf(msg, args...)
+				if len(trArgs) == 1 {
+					for i := 0; i < val.Len(); i++ {
+						fmtArgs = append(fmtArgs, val.Index(i).Interface())
+					}
+				} else {
+					log.Error("the args for i18n shouldn't contain uncertain slices, key=%q, args=%v", trKey, trArgs)
+					break
+				}
+			} else {
+				fmtArgs = append(fmtArgs, arg)
+			}
+		}
+		return fmt.Sprintf(trMsg, fmtArgs...)
+	}
+	return trMsg
+}
+
+func ResetDefaultLocales() {
+	DefaultLocales = NewLocaleStore()
+}
+
+// Tr use default locales to translate content to target language.
+func Tr(lang, trKey string, trArgs ...interface{}) string {
+	return DefaultLocales.Tr(lang, trKey, trArgs...)
+}
diff --git a/modules/translation/i18n/i18n_test.go b/modules/translation/i18n/i18n_test.go
new file mode 100644
index 0000000000..70066016cf
--- /dev/null
+++ b/modules/translation/i18n/i18n_test.go
@@ -0,0 +1,56 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package i18n
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_Tr(t *testing.T) {
+	testData1 := []byte(`
+.dot.name = Dot Name
+fmt = %[1]s %[2]s
+
+[section]
+sub = Sub String
+mixed = test value; <span style="color: red\; background: none;">more text</span>
+`)
+
+	testData2 := []byte(`
+fmt = %[2]s %[1]s
+
+[section]
+sub = Changed Sub String
+`)
+
+	ls := NewLocaleStore()
+	assert.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", testData1))
+	assert.NoError(t, ls.AddLocaleByIni("lang2", "Lang2", testData2))
+	ls.SetDefaultLang("lang1")
+
+	result := ls.Tr("lang1", "fmt", "a", "b")
+	assert.Equal(t, "a b", result)
+
+	result = ls.Tr("lang2", "fmt", "a", "b")
+	assert.Equal(t, "b a", result)
+
+	result = ls.Tr("lang1", "section.sub")
+	assert.Equal(t, "Sub String", result)
+
+	result = ls.Tr("lang2", "section.sub")
+	assert.Equal(t, "Changed Sub String", result)
+
+	result = ls.Tr("", ".dot.name")
+	assert.Equal(t, "Dot Name", result)
+
+	result = ls.Tr("lang2", "section.mixed")
+	assert.Equal(t, `test value; <span style="color: red; background: none;">more text</span>`, result)
+
+	langs, descs := ls.ListLangNameDesc()
+	assert.Equal(t, []string{"lang1", "lang2"}, langs)
+	assert.Equal(t, []string{"Lang1", "Lang2"}, descs)
+}
diff --git a/modules/translation/translation.go b/modules/translation/translation.go
index fd38e4d510..da9d9b9b68 100644
--- a/modules/translation/translation.go
+++ b/modules/translation/translation.go
@@ -11,8 +11,8 @@ import (
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/options"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/translation/i18n"
 
-	"github.com/unknwon/i18n"
 	"golang.org/x/text/language"
 )
 
@@ -54,13 +54,13 @@ func TryTr(lang, format string, args ...interface{}) (string, bool) {
 
 // InitLocales loads the locales
 func InitLocales() {
-	i18n.Reset()
+	i18n.ResetDefaultLocales()
 	localeNames, err := options.Dir("locale")
 	if err != nil {
 		log.Fatal("Failed to list locale files: %v", err)
 	}
 
-	localFiles := make(map[string][]byte)
+	localFiles := make(map[string][]byte, len(localeNames))
 	for _, name := range localeNames {
 		localFiles[name], err = options.Locale(name)
 		if err != nil {
@@ -76,16 +76,21 @@ func InitLocales() {
 	matcher = language.NewMatcher(supportedTags)
 	for i := range setting.Names {
 		key := "locale_" + setting.Langs[i] + ".ini"
-		if err = i18n.SetMessageWithDesc(setting.Langs[i], setting.Names[i], localFiles[key]); err != nil {
+		if err = i18n.DefaultLocales.AddLocaleByIni(setting.Langs[i], setting.Names[i], localFiles[key]); err != nil {
 			log.Error("Failed to set messages to %s: %v", setting.Langs[i], err)
 		}
 	}
-	i18n.SetDefaultLang("en-US")
+	if len(setting.Langs) != 0 {
+		defaultLangName := setting.Langs[0]
+		if defaultLangName != "en-US" {
+			log.Info("Use the first locale (%s) in LANGS setting option as default", defaultLangName)
+		}
+		i18n.DefaultLocales.SetDefaultLang(defaultLangName)
+	}
 
-	allLangs = make([]*LangType, 0, i18n.Count())
+	langs, descs := i18n.DefaultLocales.ListLangNameDesc()
+	allLangs = make([]*LangType, 0, len(langs))
 	allLangMap = map[string]*LangType{}
-	langs := i18n.ListLangs()
-	descs := i18n.ListLangDescs()
 	for i, v := range langs {
 		l := &LangType{v, descs[i]}
 		allLangs = append(allLangs, l)