diff options
Diffstat (limited to 'modules')
| -rw-r--r-- | modules/base/tool.go | 4 | ||||
| -rw-r--r-- | modules/template/template.go | 10 |
2 files changed, 8 insertions, 6 deletions
diff --git a/modules/base/tool.go b/modules/base/tool.go index 46eeca9b2b..9dcbb881d5 100644 --- a/modules/base/tool.go +++ b/modules/base/tool.go @@ -25,6 +25,7 @@ import ( "golang.org/x/net/html/charset" "github.com/gogits/gogs/modules/avatar" + "github.com/gogits/gogs/modules/log" "github.com/gogits/gogs/modules/setting" ) @@ -52,7 +53,8 @@ func ShortSha(sha1 string) string { } func DetectEncoding(content []byte) string { - _, name, _ := charset.DetermineEncoding(content, setting.Repository.AnsiCharset) + _, name, certain := charset.DetermineEncoding(content, setting.Repository.AnsiCharset) + log.Debug("Detected encoding: %s (%v)", name, certain) return name } diff --git a/modules/template/template.go b/modules/template/template.go index 638febe89d..4149de451a 100644 --- a/modules/template/template.go +++ b/modules/template/template.go @@ -12,6 +12,7 @@ import ( "runtime" "strings" "time" + "unicode/utf8" "golang.org/x/net/html/charset" "golang.org/x/text/transform" @@ -130,20 +131,19 @@ func Sha1(str string) string { } func ToUtf8WithErr(content []byte) (error, string) { - charsetLabel := base.DetectEncoding(content) - if charsetLabel == "UTF-8" { + if utf8.Valid(content[:1024]) { return nil, string(content) } + charsetLabel := base.DetectEncoding(content) encoding, _ := charset.Lookup(charsetLabel) if encoding == nil { - return fmt.Errorf("unknown char decoder %s", charsetLabel), string(content) + return fmt.Errorf("Unknown encoding: %s", charsetLabel), string(content) } - result, n, err := transform.String(encoding.NewDecoder(), string(content)) - // If there is an error, we concatenate the nicely decoded part and the // original left over. This way we won't loose data. + result, n, err := transform.String(encoding.NewDecoder(), string(content)) if err != nil { result = result + string(content[n:]) } |