diff options
| author | zeripath <art27@cantab.net> | 2019-04-26 13:00:30 +0100 |
|---|---|---|
| committer | Lauris BH <lauris@nix.lv> | 2019-04-26 15:00:30 +0300 |
| commit | f6eedd4dc8fb10df869750c69e2bead0521ec0eb (patch) | |
| tree | 52ee0f6f7848be01bb3752152f81521dc4c24c33 /modules/templates | |
| parent | 4c34bc111ce020161a2fbd962a19a9123b3e2dc4 (diff) | |
| download | gitea-f6eedd4dc8fb10df869750c69e2bead0521ec0eb.tar.gz gitea-f6eedd4dc8fb10df869750c69e2bead0521ec0eb.zip | |
UI: Detect and restore encoding and BOM in content (#6727)
* detect and remove a decoded BOM
Signed-off-by: Andrew Thornton <art27@cantab.net>
* Restore the previous encoding and BOM
* On error keep as UTF-8
Signed-off-by: Andrew Thornton <art27@cantab.net>
* create remove BOM function
* Deal with LFSed content
* Update modules/repofiles/update.go
* Fix final LFS bug
* Keep LFS sections referring to opts.Content
Diffstat (limited to 'modules/templates')
| -rw-r--r-- | modules/templates/helper.go | 14 |
1 files changed, 8 insertions, 6 deletions
diff --git a/modules/templates/helper.go b/modules/templates/helper.go index d3eb8c48b8..b6c5cc5945 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -267,7 +267,7 @@ func ToUTF8WithErr(content []byte) (string, error) { if err != nil { return "", err } else if charsetLabel == "UTF-8" { - return string(content), nil + return string(base.RemoveBOMIfPresent(content)), nil } encoding, _ := charset.Lookup(charsetLabel) @@ -277,19 +277,21 @@ func ToUTF8WithErr(content []byte) (string, error) { // If there is an error, we concatenate the nicely decoded part and the // original left over. This way we won't lose data. - result, n, err := transform.String(encoding.NewDecoder(), string(content)) + result, n, err := transform.Bytes(encoding.NewDecoder(), content) if err != nil { - result = result + string(content[n:]) + result = append(result, content[n:]...) } - return result, err + result = base.RemoveBOMIfPresent(result) + + return string(result), err } // ToUTF8WithFallback detects the encoding of content and coverts to UTF-8 if possible func ToUTF8WithFallback(content []byte) []byte { charsetLabel, err := base.DetectEncoding(content) if err != nil || charsetLabel == "UTF-8" { - return content + return base.RemoveBOMIfPresent(content) } encoding, _ := charset.Lookup(charsetLabel) @@ -304,7 +306,7 @@ func ToUTF8WithFallback(content []byte) []byte { return append(result, content[n:]...) } - return result + return base.RemoveBOMIfPresent(result) } // ToUTF8 converts content to UTF8 encoding and ignore error |