Convert files to utf-8 for indexing (#7814)

* Convert files to utf-8 for indexing

* Move utf8 functions to modules/base

* Bump repoIndexerLatestVersion to 3

* Add tests for base/encoding.go

* Changes to pass gosimple

* Move UTF8 funcs into new modules/charset package

13 files changed, 371 insertions, 166 deletions

diff --git a/integrations/migration-test/migration_test.go b/integrations/migration-test/migration_test.go
index fd0d0b7c72..858f339b89 100644
--- a/integrations/migration-test/migration_test.go
+++ b/integrations/migration-test/migration_test.go
@@ -20,6 +20,7 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/models/migrations"
 	"code.gitea.io/gitea/modules/base"
+	"code.gitea.io/gitea/modules/charset"
 	"code.gitea.io/gitea/modules/setting"
 
 	"github.com/go-xorm/xorm"
@@ -106,7 +107,7 @@ func readSQLFromFile(version string) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	return string(base.RemoveBOMIfPresent(bytes)), nil
+	return string(charset.RemoveBOMIfPresent(bytes)), nil
 }
 
 func restoreOldDB(t *testing.T, version string) bool {
diff --git a/models/git_diff.go b/models/git_diff.go
index 518d543e7b..62bb35be07 100644
--- a/models/git_diff.go
+++ b/models/git_diff.go
@@ -19,7 +19,7 @@ import (
 	"strconv"
 	"strings"
 
-	"code.gitea.io/gitea/modules/base"
+	"code.gitea.io/gitea/modules/charset"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/highlight"
 	"code.gitea.io/gitea/modules/log"
@@ -27,7 +27,7 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 	"github.com/Unknwon/com"
 	"github.com/sergi/go-diff/diffmatchpatch"
-	"golang.org/x/net/html/charset"
+	stdcharset "golang.org/x/net/html/charset"
 	"golang.org/x/text/transform"
 )
 
@@ -641,9 +641,9 @@ func ParsePatch(maxLines, maxLineCharacters, maxFiles int, reader io.Reader) (*D
 				buf.WriteString("\n")
 			}
 		}
-		charsetLabel, err := base.DetectEncoding(buf.Bytes())
+		charsetLabel, err := charset.DetectEncoding(buf.Bytes())
 		if charsetLabel != "UTF-8" && err == nil {
-			encoding, _ := charset.Lookup(charsetLabel)
+			encoding, _ := stdcharset.Lookup(charsetLabel)
 			if encoding != nil {
 				d := encoding.NewDecoder()
 				for _, sec := range f.Sections {
diff --git a/models/repo_indexer.go b/models/repo_indexer.go
index c991b1aac9..f625b80389 100644
--- a/models/repo_indexer.go
+++ b/models/repo_indexer.go
@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"code.gitea.io/gitea/modules/base"
+	"code.gitea.io/gitea/modules/charset"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/indexer"
 	"code.gitea.io/gitea/modules/log"
@@ -207,6 +208,7 @@ func addUpdate(update fileUpdate, repo *Repository, batch rupture.FlushingBatch)
 	if err != nil {
 		return err
 	} else if !base.IsTextFile(fileContents) {
+		// FIXME: UTF-16 files will probably fail here
 		return nil
 	}
 	indexerUpdate := indexer.RepoIndexerUpdate{
@@ -214,7 +216,7 @@ func addUpdate(update fileUpdate, repo *Repository, batch rupture.FlushingBatch)
 		Op:       indexer.RepoIndexerOpUpdate,
 		Data: &indexer.RepoIndexerData{
 			RepoID:  repo.ID,
-			Content: string(fileContents),
+			Content: string(charset.ToUTF8DropErrors(fileContents)),
 		},
 	}
 	return indexerUpdate.AddToFlushingBatch(batch)
diff --git a/modules/base/tool.go b/modules/base/tool.go
index 4893abff71..2bb39dbfea 100644
--- a/modules/base/tool.go
+++ b/modules/base/tool.go
@@ -5,7 +5,6 @@
 package base
 
 import (
-	"bytes"
 	"crypto/md5"
 	"crypto/rand"
 	"crypto/sha1"
@@ -26,7 +25,6 @@ import (
 	"strings"
 	"time"
 	"unicode"
-	"unicode/utf8"
 
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
@@ -35,12 +33,8 @@ import (
 
 	"github.com/Unknwon/com"
 	"github.com/Unknwon/i18n"
-	"github.com/gogits/chardet"
 )
 
-// UTF8BOM is the utf-8 byte-order marker
-var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'}
-
 // EncodeMD5 encodes string to md5 hex value.
 func EncodeMD5(str string) string {
 	m := md5.New()
@@ -68,49 +62,6 @@ func ShortSha(sha1 string) string {
 	return TruncateString(sha1, 10)
 }
 
-// DetectEncoding detect the encoding of content
-func DetectEncoding(content []byte) (string, error) {
-	if utf8.Valid(content) {
-		log.Debug("Detected encoding: utf-8 (fast)")
-		return "UTF-8", nil
-	}
-
-	textDetector := chardet.NewTextDetector()
-	var detectContent []byte
-	if len(content) < 1024 {
-		// Check if original content is valid
-		if _, err := textDetector.DetectBest(content); err != nil {
-			return "", err
-		}
-		times := 1024 / len(content)
-		detectContent = make([]byte, 0, times*len(content))
-		for i := 0; i < times; i++ {
-			detectContent = append(detectContent, content...)
-		}
-	} else {
-		detectContent = content
-	}
-	result, err := textDetector.DetectBest(detectContent)
-	if err != nil {
-		return "", err
-	}
-	if result.Charset != "UTF-8" && len(setting.Repository.AnsiCharset) > 0 {
-		log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset)
-		return setting.Repository.AnsiCharset, err
-	}
-
-	log.Debug("Detected encoding: %s", result.Charset)
-	return result.Charset, err
-}
-
-// RemoveBOMIfPresent removes a UTF-8 BOM from a []byte
-func RemoveBOMIfPresent(content []byte) []byte {
-	if len(content) > 2 && bytes.Equal(content[0:3], UTF8BOM) {
-		return content[3:]
-	}
-	return content
-}
-
 // BasicAuthDecode decode basic auth string
 func BasicAuthDecode(encoded string) (string, string, error) {
 	s, err := base64.StdEncoding.DecodeString(encoded)
diff --git a/modules/base/tool_test.go b/modules/base/tool_test.go
index fa61e5dfb1..3f1eebfc4b 100644
--- a/modules/base/tool_test.go
+++ b/modules/base/tool_test.go
@@ -64,42 +64,6 @@ func TestShortSha(t *testing.T) {
 	assert.Equal(t, "veryverylo", ShortSha("veryverylong"))
 }
 
-func TestDetectEncoding(t *testing.T) {
-	testSuccess := func(b []byte, expected string) {
-		encoding, err := DetectEncoding(b)
-		assert.NoError(t, err)
-		assert.Equal(t, expected, encoding)
-	}
-	// utf-8
-	b := []byte("just some ascii")
-	testSuccess(b, "UTF-8")
-
-	// utf-8-sig: "hey" (with BOM)
-	b = []byte{0xef, 0xbb, 0xbf, 0x68, 0x65, 0x79}
-	testSuccess(b, "UTF-8")
-
-	// utf-16: "hey<accented G>"
-	b = []byte{0xff, 0xfe, 0x68, 0x00, 0x65, 0x00, 0x79, 0x00, 0xf4, 0x01}
-	testSuccess(b, "UTF-16LE")
-
-	// iso-8859-1: d<accented e>cor<newline>
-	b = []byte{0x44, 0xe9, 0x63, 0x6f, 0x72, 0x0a}
-	encoding, err := DetectEncoding(b)
-	assert.NoError(t, err)
-	// due to a race condition in `chardet` library, it could either detect
-	// "ISO-8859-1" or "IS0-8859-2" here. Technically either is correct, so
-	// we accept either.
-	assert.Contains(t, encoding, "ISO-8859")
-
-	setting.Repository.AnsiCharset = "placeholder"
-	testSuccess(b, "placeholder")
-
-	// invalid bytes
-	b = []byte{0xfa}
-	_, err = DetectEncoding(b)
-	assert.Error(t, err)
-}
-
 func TestBasicAuthDecode(t *testing.T) {
 	_, _, err := BasicAuthDecode("?")
 	assert.Equal(t, "illegal base64 data at input byte 0", err.Error())
diff --git a/modules/charset/charset.go b/modules/charset/charset.go
new file mode 100644
index 0000000000..f43fe387ed
--- /dev/null
+++ b/modules/charset/charset.go
@@ -0,0 +1,152 @@
+// Copyright 2014 The Gogs Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package charset
+
+import (
+	"bytes"
+	"fmt"
+	"unicode/utf8"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/gogits/chardet"
+	"golang.org/x/net/html/charset"
+	"golang.org/x/text/transform"
+)
+
+// UTF8BOM is the utf-8 byte-order marker
+var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'}
+
+// ToUTF8WithErr converts content to UTF8 encoding
+func ToUTF8WithErr(content []byte) (string, error) {
+	charsetLabel, err := DetectEncoding(content)
+	if err != nil {
+		return "", err
+	} else if charsetLabel == "UTF-8" {
+		return string(RemoveBOMIfPresent(content)), nil
+	}
+
+	encoding, _ := charset.Lookup(charsetLabel)
+	if encoding == nil {
+		return string(content), fmt.Errorf("Unknown encoding: %s", charsetLabel)
+	}
+
+	// If there is an error, we concatenate the nicely decoded part and the
+	// original left over. This way we won't lose data.
+	result, n, err := transform.Bytes(encoding.NewDecoder(), content)
+	if err != nil {
+		result = append(result, content[n:]...)
+	}
+
+	result = RemoveBOMIfPresent(result)
+
+	return string(result), err
+}
+
+// ToUTF8WithFallback detects the encoding of content and coverts to UTF-8 if possible
+func ToUTF8WithFallback(content []byte) []byte {
+	charsetLabel, err := DetectEncoding(content)
+	if err != nil || charsetLabel == "UTF-8" {
+		return RemoveBOMIfPresent(content)
+	}
+
+	encoding, _ := charset.Lookup(charsetLabel)
+	if encoding == nil {
+		return content
+	}
+
+	// If there is an error, we concatenate the nicely decoded part and the
+	// original left over. This way we won't lose data.
+	result, n, err := transform.Bytes(encoding.NewDecoder(), content)
+	if err != nil {
+		return append(result, content[n:]...)
+	}
+
+	return RemoveBOMIfPresent(result)
+}
+
+// ToUTF8 converts content to UTF8 encoding and ignore error
+func ToUTF8(content string) string {
+	res, _ := ToUTF8WithErr([]byte(content))
+	return res
+}
+
+// ToUTF8DropErrors makes sure the return string is valid utf-8; attempts conversion if possible
+func ToUTF8DropErrors(content []byte) []byte {
+	charsetLabel, err := DetectEncoding(content)
+	if err != nil || charsetLabel == "UTF-8" {
+		return RemoveBOMIfPresent(content)
+	}
+
+	encoding, _ := charset.Lookup(charsetLabel)
+	if encoding == nil {
+		return content
+	}
+
+	// We ignore any non-decodable parts from the file.
+	// Some parts might be lost
+	var decoded []byte
+	decoder := encoding.NewDecoder()
+	idx := 0
+	for {
+		result, n, err := transform.Bytes(decoder, content[idx:])
+		decoded = append(decoded, result...)
+		if err == nil {
+			break
+		}
+		decoded = append(decoded, ' ')
+		idx = idx + n + 1
+		if idx >= len(content) {
+			break
+		}
+	}
+
+	return RemoveBOMIfPresent(decoded)
+}
+
+// RemoveBOMIfPresent removes a UTF-8 BOM from a []byte
+func RemoveBOMIfPresent(content []byte) []byte {
+	if len(content) > 2 && bytes.Equal(content[0:3], UTF8BOM) {
+		return content[3:]
+	}
+	return content
+}
+
+// DetectEncoding detect the encoding of content
+func DetectEncoding(content []byte) (string, error) {
+	if utf8.Valid(content) {
+		log.Debug("Detected encoding: utf-8 (fast)")
+		return "UTF-8", nil
+	}
+
+	textDetector := chardet.NewTextDetector()
+	var detectContent []byte
+	if len(content) < 1024 {
+		// Check if original content is valid
+		if _, err := textDetector.DetectBest(content); err != nil {
+			return "", err
+		}
+		times := 1024 / len(content)
+		detectContent = make([]byte, 0, times*len(content))
+		for i := 0; i < times; i++ {
+			detectContent = append(detectContent, content...)
+		}
+	} else {
+		detectContent = content
+	}
+	result, err := textDetector.DetectBest(detectContent)
+	if err != nil {
+		return "", err
+	}
+	// FIXME: to properly decouple this function the fallback ANSI charset should be passed as an argument
+	if result.Charset != "UTF-8" && len(setting.Repository.AnsiCharset) > 0 {
+		log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset)
+		return setting.Repository.AnsiCharset, err
+	}
+
+	log.Debug("Detected encoding: %s", result.Charset)
+	return result.Charset, err
+}
diff --git a/modules/charset/charset_test.go b/modules/charset/charset_test.go
new file mode 100644
index 0000000000..fc9ec6a979
--- /dev/null
+++ b/modules/charset/charset_test.go
@@ -0,0 +1,191 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package charset
+
+import (
+	"testing"
+
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRemoveBOMIfPresent(t *testing.T) {
+	res := RemoveBOMIfPresent([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
+	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
+
+	res = RemoveBOMIfPresent([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
+	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
+}
+
+func TestToUTF8WithErr(t *testing.T) {
+	var res string
+	var err error
+
+	res, err = ToUTF8WithErr([]byte{0x41, 0x42, 0x43})
+	assert.Equal(t, "ABC", res)
+	assert.NoError(t, err)
+
+	res, err = ToUTF8WithErr([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
+	assert.Equal(t, "áéíóú", res)
+	assert.NoError(t, err)
+
+	res, err = ToUTF8WithErr([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
+	assert.Equal(t, "áéíóú", res)
+	assert.NoError(t, err)
+
+	// This test FAILS
+	res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73})
+	assert.Equal(t, "Hola, así cómo ños", res)
+	assert.NoError(t, err)
+
+	res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73})
+	// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
+	assert.Regexp(t, "^Hola, así cómo", res)
+	assert.NoError(t, err)
+
+	res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73})
+	// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
+	assert.Regexp(t, "^Hola, así cómo", res)
+	assert.NoError(t, err)
+
+	// Japanese (Shift-JIS)
+	res, err = ToUTF8WithErr([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42})
+	assert.Equal(t, "日属秘ぞしちゅ。", res)
+	assert.NoError(t, err)
+
+	res, err = ToUTF8WithErr([]byte{0x00, 0x00, 0x00, 0x00})
+	assert.Equal(t, "\x00\x00\x00\x00", res)
+	assert.NoError(t, err)
+}
+
+func TestToUTF8WithFallback(t *testing.T) {
+	res := ToUTF8WithFallback([]byte{0x41, 0x42, 0x43})
+	assert.Equal(t, []byte("ABC"), res)
+
+	res = ToUTF8WithFallback([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
+	assert.Equal(t, []byte("áéíóú"), res)
+
+	res = ToUTF8WithFallback([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
+	assert.Equal(t, []byte("áéíóú"), res)
+
+	res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73})
+	assert.Equal(t, []byte("Hola, así cómo ños"), res)
+
+	minmatch := []byte("Hola, así cómo ")
+
+	res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73})
+	// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
+	assert.Equal(t, minmatch, res[0:len(minmatch)])
+
+	res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73})
+	// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
+	assert.Equal(t, minmatch, res[0:len(minmatch)])
+
+	// Japanese (Shift-JIS)
+	res = ToUTF8WithFallback([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42})
+	assert.Equal(t, []byte("日属秘ぞしちゅ。"), res)
+
+	res = ToUTF8WithFallback([]byte{0x00, 0x00, 0x00, 0x00})
+	assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res)
+}
+
+func TestToUTF8(t *testing.T) {
+	res := ToUTF8("ABC")
+	assert.Equal(t, "ABC", res)
+
+	res = ToUTF8("áéíóú")
+	assert.Equal(t, "áéíóú", res)
+
+	// With utf-8 BOM
+	res = ToUTF8("\ufeffáéíóú")
+	assert.Equal(t, "áéíóú", res)
+
+	res = ToUTF8("Hola, así cómo ños")
+	assert.Equal(t, "Hola, así cómo ños", res)
+
+	res = ToUTF8("Hola, así cómo \x07ños")
+	// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
+	assert.Regexp(t, "^Hola, así cómo", res)
+
+	// This test FAILS
+	// res = ToUTF8("Hola, así cómo \x81ños")
+	// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
+	// assert.Regexp(t, "^Hola, así cómo", res)
+
+	// Japanese (Shift-JIS)
+	res = ToUTF8("\x93\xFA\x91\xAE\x94\xE9\x82\xBC\x82\xB5\x82\xBF\x82\xE3\x81\x42")
+	assert.Equal(t, "日属秘ぞしちゅ。", res)
+
+	res = ToUTF8("\x00\x00\x00\x00")
+	assert.Equal(t, "\x00\x00\x00\x00", res)
+}
+
+func TestToUTF8DropErrors(t *testing.T) {
+	res := ToUTF8DropErrors([]byte{0x41, 0x42, 0x43})
+	assert.Equal(t, []byte("ABC"), res)
+
+	res = ToUTF8DropErrors([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
+	assert.Equal(t, []byte("áéíóú"), res)
+
+	res = ToUTF8DropErrors([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
+	assert.Equal(t, []byte("áéíóú"), res)
+
+	res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73})
+	assert.Equal(t, []byte("Hola, así cómo ños"), res)
+
+	minmatch := []byte("Hola, así cómo ")
+
+	res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73})
+	// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
+	assert.Equal(t, minmatch, res[0:len(minmatch)])
+
+	res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73})
+	// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
+	assert.Equal(t, minmatch, res[0:len(minmatch)])
+
+	// Japanese (Shift-JIS)
+	res = ToUTF8DropErrors([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42})
+	assert.Equal(t, []byte("日属秘ぞしちゅ。"), res)
+
+	res = ToUTF8DropErrors([]byte{0x00, 0x00, 0x00, 0x00})
+	assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res)
+}
+
+func TestDetectEncoding(t *testing.T) {
+	testSuccess := func(b []byte, expected string) {
+		encoding, err := DetectEncoding(b)
+		assert.NoError(t, err)
+		assert.Equal(t, expected, encoding)
+	}
+	// utf-8
+	b := []byte("just some ascii")
+	testSuccess(b, "UTF-8")
+
+	// utf-8-sig: "hey" (with BOM)
+	b = []byte{0xef, 0xbb, 0xbf, 0x68, 0x65, 0x79}
+	testSuccess(b, "UTF-8")
+
+	// utf-16: "hey<accented G>"
+	b = []byte{0xff, 0xfe, 0x68, 0x00, 0x65, 0x00, 0x79, 0x00, 0xf4, 0x01}
+	testSuccess(b, "UTF-16LE")
+
+	// iso-8859-1: d<accented e>cor<newline>
+	b = []byte{0x44, 0xe9, 0x63, 0x6f, 0x72, 0x0a}
+	encoding, err := DetectEncoding(b)
+	assert.NoError(t, err)
+	// due to a race condition in `chardet` library, it could either detect
+	// "ISO-8859-1" or "IS0-8859-2" here. Technically either is correct, so
+	// we accept either.
+	assert.Contains(t, encoding, "ISO-8859")
+
+	setting.Repository.AnsiCharset = "placeholder"
+	testSuccess(b, "placeholder")
+
+	// invalid bytes
+	b = []byte{0xfa}
+	_, err = DetectEncoding(b)
+	assert.Error(t, err)
+}
diff --git a/modules/indexer/repo.go b/modules/indexer/repo.go
index 787f9a9467..645a6fc88b 100644
--- a/modules/indexer/repo.go
+++ b/modules/indexer/repo.go
@@ -23,7 +23,7 @@ const (
 	repoIndexerAnalyzer = "repoIndexerAnalyzer"
 	repoIndexerDocType  = "repoIndexerDocType"
 
-	repoIndexerLatestVersion = 2
+	repoIndexerLatestVersion = 3
 )
 
 // repoIndexer (thread-safe) index for repository contents
diff --git a/modules/repofiles/update.go b/modules/repofiles/update.go
index 62e2afbc96..ee1b16bce9 100644
--- a/modules/repofiles/update.go
+++ b/modules/repofiles/update.go
@@ -11,17 +11,17 @@ import (
 	"path"
 	"strings"
 
-	"golang.org/x/net/html/charset"
-	"golang.org/x/text/transform"
-
 	"code.gitea.io/gitea/models"
-	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/cache"
+	"code.gitea.io/gitea/modules/charset"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/structs"
+
+	stdcharset "golang.org/x/net/html/charset"
+	"golang.org/x/text/transform"
 )
 
 // IdentityOptions for a person's identity like an author or committer
@@ -87,15 +87,15 @@ func detectEncodingAndBOM(entry *git.TreeEntry, repo *models.Repository) (string
 
 	}
 
-	encoding, err := base.DetectEncoding(buf)
+	encoding, err := charset.DetectEncoding(buf)
 	if err != nil {
 		// just default to utf-8 and no bom
 		return "UTF-8", false
 	}
 	if encoding == "UTF-8" {
-		return encoding, bytes.Equal(buf[0:3], base.UTF8BOM)
+		return encoding, bytes.Equal(buf[0:3], charset.UTF8BOM)
 	}
-	charsetEncoding, _ := charset.Lookup(encoding)
+	charsetEncoding, _ := stdcharset.Lookup(encoding)
 	if charsetEncoding == nil {
 		return "UTF-8", false
 	}
@@ -107,7 +107,7 @@ func detectEncodingAndBOM(entry *git.TreeEntry, repo *models.Repository) (string
 	}
 
 	if n > 2 {
-		return encoding, bytes.Equal([]byte(result)[0:3], base.UTF8BOM)
+		return encoding, bytes.Equal([]byte(result)[0:3], charset.UTF8BOM)
 	}
 
 	return encoding, false
@@ -321,10 +321,10 @@ func CreateOrUpdateRepoFile(repo *models.Repository, doer *models.User, opts *Up
 
 	content := opts.Content
 	if bom {
-		content = string(base.UTF8BOM) + content
+		content = string(charset.UTF8BOM) + content
 	}
 	if encoding != "UTF-8" {
-		charsetEncoding, _ := charset.Lookup(encoding)
+		charsetEncoding, _ := stdcharset.Lookup(encoding)
 		if charsetEncoding != nil {
 			result, _, err := transform.String(charsetEncoding.NewEncoder(), content)
 			if err != nil {
diff --git a/modules/templates/helper.go b/modules/templates/helper.go
index ceb4501c5b..17df375a80 100644
--- a/modules/templates/helper.go
+++ b/modules/templates/helper.go
@@ -28,8 +28,6 @@ import (
 	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/setting"
 
-	"golang.org/x/net/html/charset"
-	"golang.org/x/text/transform"
 	"gopkg.in/editorconfig/editorconfig-core-go.v1"
 )
 
@@ -274,60 +272,6 @@ func Sha1(str string) string {
 	return base.EncodeSha1(str)
 }
 
-// ToUTF8WithErr converts content to UTF8 encoding
-func ToUTF8WithErr(content []byte) (string, error) {
-	charsetLabel, err := base.DetectEncoding(content)
-	if err != nil {
-		return "", err
-	} else if charsetLabel == "UTF-8" {
-		return string(base.RemoveBOMIfPresent(content)), nil
-	}
-
-	encoding, _ := charset.Lookup(charsetLabel)
-	if encoding == nil {
-		return string(content), fmt.Errorf("Unknown encoding: %s", charsetLabel)
-	}
-
-	// If there is an error, we concatenate the nicely decoded part and the
-	// original left over. This way we won't lose data.
-	result, n, err := transform.Bytes(encoding.NewDecoder(), content)
-	if err != nil {
-		result = append(result, content[n:]...)
-	}
-
-	result = base.RemoveBOMIfPresent(result)
-
-	return string(result), err
-}
-
-// ToUTF8WithFallback detects the encoding of content and coverts to UTF-8 if possible
-func ToUTF8WithFallback(content []byte) []byte {
-	charsetLabel, err := base.DetectEncoding(content)
-	if err != nil || charsetLabel == "UTF-8" {
-		return base.RemoveBOMIfPresent(content)
-	}
-
-	encoding, _ := charset.Lookup(charsetLabel)
-	if encoding == nil {
-		return content
-	}
-
-	// If there is an error, we concatenate the nicely decoded part and the
-	// original left over. This way we won't lose data.
-	result, n, err := transform.Bytes(encoding.NewDecoder(), content)
-	if err != nil {
-		return append(result, content[n:]...)
-	}
-
-	return base.RemoveBOMIfPresent(result)
-}
-
-// ToUTF8 converts content to UTF8 encoding and ignore error
-func ToUTF8(content string) string {
-	res, _ := ToUTF8WithErr([]byte(content))
-	return res
-}
-
 // ReplaceLeft replaces all prefixes 'oldS' in 's' with 'newS'.
 func ReplaceLeft(s, oldS, newS string) string {
 	oldLen, newLen, i, n := len(oldS), len(newS), 0, 0
diff --git a/routers/repo/commit.go b/routers/repo/commit.go
index 4dbedea2a0..79fa024dc9 100644
--- a/routers/repo/commit.go
+++ b/routers/repo/commit.go
@@ -11,11 +11,11 @@ import (
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/base"
+	"code.gitea.io/gitea/modules/charset"
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
-	"code.gitea.io/gitea/modules/templates"
 )
 
 const (
@@ -251,7 +251,7 @@ func Diff(ctx *context.Context) {
 	note := &git.Note{}
 	err = git.GetNote(ctx.Repo.GitRepo, commitID, note)
 	if err == nil {
-		ctx.Data["Note"] = string(templates.ToUTF8WithFallback(note.Message))
+		ctx.Data["Note"] = string(charset.ToUTF8WithFallback(note.Message))
 		ctx.Data["NoteCommit"] = note.Commit
 		ctx.Data["NoteAuthor"] = models.ValidateCommitWithEmail(note.Commit)
 	}
diff --git a/routers/repo/editor.go b/routers/repo/editor.go
index 1c7eee98ac..68b94662dc 100644
--- a/routers/repo/editor.go
+++ b/routers/repo/editor.go
@@ -14,12 +14,12 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/auth"
 	"code.gitea.io/gitea/modules/base"
+	"code.gitea.io/gitea/modules/charset"
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/repofiles"
 	"code.gitea.io/gitea/modules/setting"
-	"code.gitea.io/gitea/modules/templates"
 	"code.gitea.io/gitea/modules/upload"
 	"code.gitea.io/gitea/modules/util"
 )
@@ -118,7 +118,7 @@ func editFile(ctx *context.Context, isNewFile bool) {
 
 		d, _ := ioutil.ReadAll(dataRc)
 		buf = append(buf, d...)
-		if content, err := templates.ToUTF8WithErr(buf); err != nil {
+		if content, err := charset.ToUTF8WithErr(buf); err != nil {
 			log.Error("ToUTF8WithErr: %v", err)
 			ctx.Data["FileContent"] = string(buf)
 		} else {
diff --git a/routers/repo/view.go b/routers/repo/view.go
index edaf24017c..b534ae0e16 100644
--- a/routers/repo/view.go
+++ b/routers/repo/view.go
@@ -16,6 +16,7 @@ import (
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/base"
+	"code.gitea.io/gitea/modules/charset"
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/highlight"
@@ -23,7 +24,6 @@ import (
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/setting"
-	"code.gitea.io/gitea/modules/templates"
 )
 
 const (
@@ -160,7 +160,7 @@ func renderDirectory(ctx *context.Context, treeLink string) {
 				ctx.Data["FileSize"] = fileSize
 			} else {
 				d, _ := ioutil.ReadAll(dataRc)
-				buf = templates.ToUTF8WithFallback(append(buf, d...))
+				buf = charset.ToUTF8WithFallback(append(buf, d...))
 
 				if markup.Type(readmeFile.Name()) != "" {
 					ctx.Data["IsMarkup"] = true
@@ -278,7 +278,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
 		}
 
 		d, _ := ioutil.ReadAll(dataRc)
-		buf = templates.ToUTF8WithFallback(append(buf, d...))
+		buf = charset.ToUTF8WithFallback(append(buf, d...))
 
 		readmeExist := markup.IsReadmeFile(blob.Name())
 		ctx.Data["ReadmeExist"] = readmeExist
@@ -293,7 +293,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
 		} else {
 			// Building code view blocks with line number on server side.
 			var fileContent string
-			if content, err := templates.ToUTF8WithErr(buf); err != nil {
+			if content, err := charset.ToUTF8WithErr(buf); err != nil {
 				log.Error("ToUTF8WithErr: %v", err)
 				fileContent = string(buf)
 			} else {