#2282 fast detection of utf-8

2 files changed, 8 insertions, 3 deletions

diff --git a/modules/base/tool.go b/modules/base/tool.go
index 75f475fa4f..5927dad01c 100644
--- a/modules/base/tool.go
+++ b/modules/base/tool.go
@@ -18,6 +18,7 @@ import (
 	"regexp"
 	"strings"
 	"time"
+	"unicode/utf8"
 
 	"github.com/Unknwon/com"
 	"github.com/Unknwon/i18n"
@@ -53,6 +54,11 @@ func ShortSha(sha1 string) string {
 }
 
 func DetectEncoding(content []byte) string {
+	if utf8.Valid(content[:1024]) {
+		log.Debug("Detected encoding: utf-8 (fast)")
+		return "utf-8"
+	}
+
 	_, name, certain := charset.DetermineEncoding(content, "")
 	if name != "utf-8" && len(setting.Repository.AnsiCharset) > 0 {
 		log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset)
diff --git a/modules/template/template.go b/modules/template/template.go
index 4149de451a..6c070b7009 100644
--- a/modules/template/template.go
+++ b/modules/template/template.go
@@ -12,7 +12,6 @@ import (
 	"runtime"
 	"strings"
 	"time"
-	"unicode/utf8"
 
 	"golang.org/x/net/html/charset"
 	"golang.org/x/text/transform"
@@ -131,11 +130,11 @@ func Sha1(str string) string {
 }
 
 func ToUtf8WithErr(content []byte) (error, string) {
-	if utf8.Valid(content[:1024]) {
+	charsetLabel := base.DetectEncoding(content)
+	if charsetLabel == "utf-8" {
 		return nil, string(content)
 	}
 
-	charsetLabel := base.DetectEncoding(content)
 	encoding, _ := charset.Lookup(charsetLabel)
 	if encoding == nil {
 		return fmt.Errorf("Unknown encoding: %s", charsetLabel), string(content)