#2282 fix utf-8 recognized as windows-1252

2 files changed, 8 insertions, 6 deletions

diff --git a/modules/base/tool.go b/modules/base/tool.go
index 46eeca9b2b..9dcbb881d5 100644
--- a/modules/base/tool.go
+++ b/modules/base/tool.go
@@ -25,6 +25,7 @@ import (
 	"golang.org/x/net/html/charset"
 
 	"github.com/gogits/gogs/modules/avatar"
+	"github.com/gogits/gogs/modules/log"
 	"github.com/gogits/gogs/modules/setting"
 )
 
@@ -52,7 +53,8 @@ func ShortSha(sha1 string) string {
 }
 
 func DetectEncoding(content []byte) string {
-	_, name, _ := charset.DetermineEncoding(content, setting.Repository.AnsiCharset)
+	_, name, certain := charset.DetermineEncoding(content, setting.Repository.AnsiCharset)
+	log.Debug("Detected encoding: %s (%v)", name, certain)
 	return name
 }
 
diff --git a/modules/template/template.go b/modules/template/template.go
index 638febe89d..4149de451a 100644
--- a/modules/template/template.go
+++ b/modules/template/template.go
@@ -12,6 +12,7 @@ import (
 	"runtime"
 	"strings"
 	"time"
+	"unicode/utf8"
 
 	"golang.org/x/net/html/charset"
 	"golang.org/x/text/transform"
@@ -130,20 +131,19 @@ func Sha1(str string) string {
 }
 
 func ToUtf8WithErr(content []byte) (error, string) {
-	charsetLabel := base.DetectEncoding(content)
-	if charsetLabel == "UTF-8" {
+	if utf8.Valid(content[:1024]) {
 		return nil, string(content)
 	}
 
+	charsetLabel := base.DetectEncoding(content)
 	encoding, _ := charset.Lookup(charsetLabel)
 	if encoding == nil {
-		return fmt.Errorf("unknown char decoder %s", charsetLabel), string(content)
+		return fmt.Errorf("Unknown encoding: %s", charsetLabel), string(content)
 	}
 
-	result, n, err := transform.String(encoding.NewDecoder(), string(content))
-
 	// If there is an error, we concatenate the nicely decoded part and the
 	// original left over. This way we won't loose data.
+	result, n, err := transform.String(encoding.NewDecoder(), string(content))
 	if err != nil {
 		result = result + string(content[n:])
 	}