Make better use of i18n (#20096)

* Prototyping

* Start work on creating offsets

* Modify tests

* Start prototyping with actual MPH

* Twiddle around

* Twiddle around comments

* Convert templates

* Fix external languages

* Fix latest translation

* Fix some test

* Tidy up code

* Use simple map

* go mod tidy

* Move back to data structure

- Uses less memory by creating for each language a map.

* Apply suggestions from code review

Co-authored-by: delvh <dev.lh@web.de>

* Add some comments

* Fix tests

* Try to fix tests

* Use en-US as defacto fallback

* Use correct slices

* refactor (#4)

* Remove TryTr, add log for missing translation key

* Refactor i18n

- Separate dev and production locale stores.
- Allow for live-reloading in dev mode.

Co-authored-by: zeripath <art27@cantab.net>

* Fix live-reloading & check for errors

* Make linter happy

* live-reload with periodic check (#5)

* Fix tests

Co-authored-by: delvh <dev.lh@web.de>
Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: zeripath <art27@cantab.net>

2 files changed, 125 insertions, 55 deletions

diff --git a/modules/translation/i18n/i18n.go b/modules/translation/i18n/i18n.go
index 664e457ecf..acce5f19fb 100644
--- a/modules/translation/i18n/i18n.go
+++ b/modules/translation/i18n/i18n.go
@@ -7,10 +7,13 @@ package i18n
 import (
 	"errors"
 	"fmt"
+	"os"
 	"reflect"
-	"strings"
+	"sync"
+	"time"
 
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
 
 	"gopkg.in/ini.v1"
 )
@@ -18,45 +21,90 @@ import (
 var (
 	ErrLocaleAlreadyExist = errors.New("lang already exists")
 
-	DefaultLocales = NewLocaleStore()
+	DefaultLocales = NewLocaleStore(true)
 )
 
 type locale struct {
 	store    *LocaleStore
 	langName string
-	langDesc string
-	messages *ini.File
+	textMap  map[int]string // the map key (idx) is generated by store's textIdxMap
+
+	sourceFileName      string
+	sourceFileInfo      os.FileInfo
+	lastReloadCheckTime time.Time
 }
 
 type LocaleStore struct {
-	// at the moment, all these fields are readonly after initialization
-	langNames   []string
-	langDescs   []string
-	localeMap   map[string]*locale
+	reloadMu *sync.Mutex // for non-prod(dev), use a mutex for live-reload. for prod, no mutex, no live-reload.
+
+	langNames []string
+	langDescs []string
+
+	localeMap  map[string]*locale
+	textIdxMap map[string]int
+
 	defaultLang string
 }
 
-func NewLocaleStore() *LocaleStore {
-	return &LocaleStore{localeMap: make(map[string]*locale)}
+func NewLocaleStore(isProd bool) *LocaleStore {
+	ls := &LocaleStore{localeMap: make(map[string]*locale), textIdxMap: make(map[string]int)}
+	if !isProd {
+		ls.reloadMu = &sync.Mutex{}
+	}
+	return ls
 }
 
 // AddLocaleByIni adds locale by ini into the store
-func (ls *LocaleStore) AddLocaleByIni(langName, langDesc string, localeFile interface{}, otherLocaleFiles ...interface{}) error {
+// if source is a string, then the file is loaded. in dev mode, the file can be live-reloaded
+// if source is a []byte, then the content is used
+func (ls *LocaleStore) AddLocaleByIni(langName, langDesc string, source interface{}) error {
 	if _, ok := ls.localeMap[langName]; ok {
 		return ErrLocaleAlreadyExist
 	}
+
+	lc := &locale{store: ls, langName: langName}
+	if fileName, ok := source.(string); ok {
+		lc.sourceFileName = fileName
+		lc.sourceFileInfo, _ = os.Stat(fileName) // live-reload only works for regular files. the error can be ignored
+	}
+
+	ls.langNames = append(ls.langNames, langName)
+	ls.langDescs = append(ls.langDescs, langDesc)
+	ls.localeMap[lc.langName] = lc
+
+	return ls.reloadLocaleByIni(langName, source)
+}
+
+func (ls *LocaleStore) reloadLocaleByIni(langName string, source interface{}) error {
 	iniFile, err := ini.LoadSources(ini.LoadOptions{
 		IgnoreInlineComment:         true,
 		UnescapeValueCommentSymbols: true,
-	}, localeFile, otherLocaleFiles...)
-	if err == nil {
-		iniFile.BlockMode = false
-		lc := &locale{store: ls, langName: langName, langDesc: langDesc, messages: iniFile}
-		ls.langNames = append(ls.langNames, lc.langName)
-		ls.langDescs = append(ls.langDescs, lc.langDesc)
-		ls.localeMap[lc.langName] = lc
+	}, source)
+	if err != nil {
+		return fmt.Errorf("unable to load ini: %w", err)
 	}
-	return err
+	iniFile.BlockMode = false
+
+	lc := ls.localeMap[langName]
+	lc.textMap = make(map[int]string)
+	for _, section := range iniFile.Sections() {
+		for _, key := range section.Keys() {
+			var trKey string
+			if section.Name() == "" || section.Name() == "DEFAULT" {
+				trKey = key.Name()
+			} else {
+				trKey = section.Name() + "." + key.Name()
+			}
+			textIdx, ok := ls.textIdxMap[trKey]
+			if !ok {
+				textIdx = len(ls.textIdxMap)
+				ls.textIdxMap[trKey] = textIdx
+			}
+			lc.textMap[textIdx] = key.Value()
+		}
+	}
+	iniFile = nil
+	return nil
 }
 
 func (ls *LocaleStore) HasLang(langName string) bool {
@@ -87,23 +135,37 @@ func (ls *LocaleStore) Tr(lang, trKey string, trArgs ...interface{}) string {
 
 // Tr translates content to locale language. fall back to default language.
 func (l *locale) Tr(trKey string, trArgs ...interface{}) string {
-	var section string
-
-	idx := strings.IndexByte(trKey, '.')
-	if idx > 0 {
-		section = trKey[:idx]
-		trKey = trKey[idx+1:]
+	if l.store.reloadMu != nil {
+		l.store.reloadMu.Lock()
+		defer l.store.reloadMu.Unlock()
+		now := time.Now()
+		if now.Sub(l.lastReloadCheckTime) >= time.Second && l.sourceFileInfo != nil && l.sourceFileName != "" {
+			l.lastReloadCheckTime = now
+			if sourceFileInfo, err := os.Stat(l.sourceFileName); err == nil && !sourceFileInfo.ModTime().Equal(l.sourceFileInfo.ModTime()) {
+				if err = l.store.reloadLocaleByIni(l.langName, l.sourceFileName); err == nil {
+					l.sourceFileInfo = sourceFileInfo
+				} else {
+					log.Error("unable to live-reload the locale file %q, err: %v", l.sourceFileName, err)
+				}
+			}
+		}
 	}
+	msg, _ := l.tryTr(trKey, trArgs...)
+	return msg
+}
 
+func (l *locale) tryTr(trKey string, trArgs ...interface{}) (msg string, found bool) {
 	trMsg := trKey
-	if trIni, err := l.messages.Section(section).GetKey(trKey); err == nil {
-		trMsg = trIni.Value()
-	} else if l.store.defaultLang != "" && l.langName != l.store.defaultLang {
-		// try to fall back to default
-		if defaultLocale, ok := l.store.localeMap[l.store.defaultLang]; ok {
-			if trIni, err = defaultLocale.messages.Section(section).GetKey(trKey); err == nil {
-				trMsg = trIni.Value()
+	textIdx, ok := l.store.textIdxMap[trKey]
+	if ok {
+		if msg, found = l.textMap[textIdx]; found {
+			trMsg = msg // use current translation
+		} else if l.langName != l.store.defaultLang {
+			if def, ok := l.store.localeMap[l.store.defaultLang]; ok {
+				return def.tryTr(trKey, trArgs...)
 			}
+		} else if !setting.IsProd {
+			log.Error("missing i18n translation key: %q", trKey)
 		}
 	}
 
@@ -128,13 +190,15 @@ func (l *locale) Tr(trKey string, trArgs ...interface{}) string {
 				fmtArgs = append(fmtArgs, arg)
 			}
 		}
-		return fmt.Sprintf(trMsg, fmtArgs...)
+		return fmt.Sprintf(trMsg, fmtArgs...), found
 	}
-	return trMsg
+	return trMsg, found
 }
 
-func ResetDefaultLocales() {
-	DefaultLocales = NewLocaleStore()
+// ResetDefaultLocales resets the current default locales
+// NOTE: this is not synchronized
+func ResetDefaultLocales(isProd bool) {
+	DefaultLocales = NewLocaleStore(isProd)
 }
 
 // Tr use default locales to translate content to target language.
diff --git a/modules/translation/i18n/i18n_test.go b/modules/translation/i18n/i18n_test.go
index 70066016cf..32f7585b32 100644
--- a/modules/translation/i18n/i18n_test.go
+++ b/modules/translation/i18n/i18n_test.go
@@ -27,30 +27,36 @@ fmt = %[2]s %[1]s
 sub = Changed Sub String
 `)
 
-	ls := NewLocaleStore()
-	assert.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", testData1))
-	assert.NoError(t, ls.AddLocaleByIni("lang2", "Lang2", testData2))
-	ls.SetDefaultLang("lang1")
+	for _, isProd := range []bool{true, false} {
+		ls := NewLocaleStore(isProd)
+		assert.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", testData1))
+		assert.NoError(t, ls.AddLocaleByIni("lang2", "Lang2", testData2))
+		ls.SetDefaultLang("lang1")
 
-	result := ls.Tr("lang1", "fmt", "a", "b")
-	assert.Equal(t, "a b", result)
+		result := ls.Tr("lang1", "fmt", "a", "b")
+		assert.Equal(t, "a b", result)
 
-	result = ls.Tr("lang2", "fmt", "a", "b")
-	assert.Equal(t, "b a", result)
+		result = ls.Tr("lang2", "fmt", "a", "b")
+		assert.Equal(t, "b a", result)
 
-	result = ls.Tr("lang1", "section.sub")
-	assert.Equal(t, "Sub String", result)
+		result = ls.Tr("lang1", "section.sub")
+		assert.Equal(t, "Sub String", result)
 
-	result = ls.Tr("lang2", "section.sub")
-	assert.Equal(t, "Changed Sub String", result)
+		result = ls.Tr("lang2", "section.sub")
+		assert.Equal(t, "Changed Sub String", result)
 
-	result = ls.Tr("", ".dot.name")
-	assert.Equal(t, "Dot Name", result)
+		result = ls.Tr("", ".dot.name")
+		assert.Equal(t, "Dot Name", result)
 
-	result = ls.Tr("lang2", "section.mixed")
-	assert.Equal(t, `test value; <span style="color: red; background: none;">more text</span>`, result)
+		result = ls.Tr("lang2", "section.mixed")
+		assert.Equal(t, `test value; <span style="color: red; background: none;">more text</span>`, result)
 
-	langs, descs := ls.ListLangNameDesc()
-	assert.Equal(t, []string{"lang1", "lang2"}, langs)
-	assert.Equal(t, []string{"Lang1", "Lang2"}, descs)
+		langs, descs := ls.ListLangNameDesc()
+		assert.Equal(t, []string{"lang1", "lang2"}, langs)
+		assert.Equal(t, []string{"Lang1", "Lang2"}, descs)
+
+		result, found := ls.localeMap["lang1"].tryTr("no-such")
+		assert.Equal(t, "no-such", result)
+		assert.False(t, found)
+	}
 }