diff options
| author | zeripath <art27@cantab.net> | 2019-04-26 13:00:30 +0100 |
|---|---|---|
| committer | Lauris BH <lauris@nix.lv> | 2019-04-26 15:00:30 +0300 |
| commit | f6eedd4dc8fb10df869750c69e2bead0521ec0eb (patch) | |
| tree | 52ee0f6f7848be01bb3752152f81521dc4c24c33 /modules/base/tool.go | |
| parent | 4c34bc111ce020161a2fbd962a19a9123b3e2dc4 (diff) | |
| download | gitea-f6eedd4dc8fb10df869750c69e2bead0521ec0eb.tar.gz gitea-f6eedd4dc8fb10df869750c69e2bead0521ec0eb.zip | |
UI: Detect and restore encoding and BOM in content (#6727)
* detect and remove a decoded BOM
Signed-off-by: Andrew Thornton <art27@cantab.net>
* Restore the previous encoding and BOM
* On error keep as UTF-8
Signed-off-by: Andrew Thornton <art27@cantab.net>
* create remove BOM function
* Deal with LFSed content
* Update modules/repofiles/update.go
* Fix final LFS bug
* Keep LFS sections referring to opts.Content
Diffstat (limited to 'modules/base/tool.go')
| -rw-r--r-- | modules/base/tool.go | 12 |
1 files changed, 12 insertions, 0 deletions
diff --git a/modules/base/tool.go b/modules/base/tool.go index 97fd87e85c..d99ed2bab4 100644 --- a/modules/base/tool.go +++ b/modules/base/tool.go @@ -5,6 +5,7 @@ package base import ( + "bytes" "crypto/md5" "crypto/rand" "crypto/sha1" @@ -36,6 +37,9 @@ import ( "github.com/gogits/chardet" ) +// UTF8BOM is the utf-8 byte-order marker +var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'} + // EncodeMD5 encodes string to md5 hex value. func EncodeMD5(str string) string { m := md5.New() @@ -91,6 +95,14 @@ func DetectEncoding(content []byte) (string, error) { return result.Charset, err } +// RemoveBOMIfPresent removes a UTF-8 BOM from a []byte +func RemoveBOMIfPresent(content []byte) []byte { + if len(content) > 2 && bytes.Equal(content[0:3], UTF8BOM) { + return content[3:] + } + return content +} + // BasicAuthDecode decode basic auth string func BasicAuthDecode(encoded string) (string, string, error) { s, err := base64.StdEncoding.DecodeString(encoded) |