Fix chardet test and add ordering option (#11621)

* Fix chardet test and add ordering option

Signed-off-by: Andrew Thornton <art27@cantab.net>

* minor fixes

Signed-off-by: Andrew Thornton <art27@cantab.net>

* remove log

Signed-off-by: Andrew Thornton <art27@cantab.net>

* remove log2

Signed-off-by: Andrew Thornton <art27@cantab.net>

* only iterate through top results

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Update docs/content/doc/advanced/config-cheat-sheet.en-us.md

* slight restructure of for loop

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: techknowlogick <techknowlogick@gitea.io>

1 files changed, 74 insertions, 0 deletions

diff --git a/modules/setting/repository.go b/modules/setting/repository.go
index 8af3eaaf46..1796a8d6b5 100644
--- a/modules/setting/repository.go
+++ b/modules/setting/repository.go
@@ -24,6 +24,8 @@ const (
 // Repository settings
 var (
 	Repository = struct {
+		DetectedCharsetsOrder                   []string
+		DetectedCharsetScore                    map[string]int `ini:"-"`
 		AnsiCharset                             string
 		ForcePrivate                            bool
 		DefaultPrivate                          string
@@ -88,6 +90,42 @@ var (
 			Wiki          []string
 		} `ini:"repository.signing"`
 	}{
+		DetectedCharsetsOrder: []string{
+			"UTF-8",
+			"UTF-16BE",
+			"UTF-16LE",
+			"UTF-32BE",
+			"UTF-32LE",
+			"ISO-8859-1",
+			"windows-1252",
+			"ISO-8859-2",
+			"windows-1250",
+			"ISO-8859-5",
+			"ISO-8859-6",
+			"ISO-8859-7",
+			"windows-1253",
+			"ISO-8859-8-I",
+			"windows-1255",
+			"ISO-8859-8",
+			"windows-1251",
+			"windows-1256",
+			"KOI8-R",
+			"ISO-8859-9",
+			"windows-1254",
+			"Shift_JIS",
+			"GB18030",
+			"EUC-JP",
+			"EUC-KR",
+			"Big5",
+			"ISO-2022-JP",
+			"ISO-2022-KR",
+			"ISO-2022-CN",
+			"IBM424_rtl",
+			"IBM424_ltr",
+			"IBM420_rtl",
+			"IBM420_ltr",
+		},
+		DetectedCharsetScore:                    map[string]int{},
 		AnsiCharset:                             "",
 		ForcePrivate:                            false,
 		DefaultPrivate:                          RepoCreatingLastUserVisibility,
@@ -208,6 +246,10 @@ func newRepository() {
 	} else {
 		RepoRootPath = filepath.Clean(RepoRootPath)
 	}
+	defaultDetectedCharsetsOrder := make([]string, 0, len(Repository.DetectedCharsetsOrder))
+	for _, charset := range Repository.DetectedCharsetsOrder {
+		defaultDetectedCharsetsOrder = append(defaultDetectedCharsetsOrder, strings.ToLower(strings.TrimSpace(charset)))
+	}
 	ScriptType = sec.Key("SCRIPT_TYPE").MustString("bash")
 
 	if err = Cfg.Section("repository").MapTo(&Repository); err != nil {
@@ -222,6 +264,38 @@ func newRepository() {
 		log.Fatal("Failed to map Repository.PullRequest settings: %v", err)
 	}
 
+	preferred := make([]string, 0, len(Repository.DetectedCharsetsOrder))
+	for _, charset := range Repository.DetectedCharsetsOrder {
+		canonicalCharset := strings.ToLower(strings.TrimSpace(charset))
+		preferred = append(preferred, canonicalCharset)
+		// remove it from the defaults
+		for i, charset := range defaultDetectedCharsetsOrder {
+			if charset == canonicalCharset {
+				defaultDetectedCharsetsOrder = append(defaultDetectedCharsetsOrder[:i], defaultDetectedCharsetsOrder[i+1:]...)
+				break
+			}
+		}
+	}
+
+	i := 0
+	for _, charset := range preferred {
+		// Add the defaults
+		if charset == "defaults" {
+			for _, charset := range defaultDetectedCharsetsOrder {
+				canonicalCharset := strings.ToLower(strings.TrimSpace(charset))
+				if _, has := Repository.DetectedCharsetScore[canonicalCharset]; !has {
+					Repository.DetectedCharsetScore[canonicalCharset] = i
+					i++
+				}
+			}
+			continue
+		}
+		if _, has := Repository.DetectedCharsetScore[charset]; !has {
+			Repository.DetectedCharsetScore[charset] = i
+			i++
+		}
+	}
+
 	if !filepath.IsAbs(Repository.Upload.TempPath) {
 		Repository.Upload.TempPath = path.Join(AppWorkPath, Repository.Upload.TempPath)
 	}