Detect charset and convert non UTF-8 files for display (#4950)

* Detect charset and convert non UTF-8 files for display

* Refactor and move function to correct module

* Revert unrelated changes

* More unrelated changes

* Duplicate content for small text to have better encoding detection

* Check if original content is valid before duplicating it

diff --git a/modules/base/tool.go b/modules/base/tool.go
index 2dfd8ffec04c36bbf9b79d7711b6b9bad7039717..d5ec9e83fcbea6a8b04a96ebdebe50b292661285 100644 (file)
--- a/modules/base/tool.go
+++ b/modules/base/tool.go
@@ -59,7 +59,22 @@ func DetectEncoding(content []byte) (string, error) {
                return "UTF-8", nil
        }
 
-       result, err := chardet.NewTextDetector().DetectBest(content)
+       textDetector := chardet.NewTextDetector()
+       var detectContent []byte
+       if len(content) < 1024 {
+               // Check if original content is valid
+               if _, err := textDetector.DetectBest(content); err != nil {
+                       return "", err
+               }
+               times := 1024 / len(content)
+               detectContent = make([]byte, 0, times*len(content))
+               for i := 0; i < times; i++ {
+                       detectContent = append(detectContent, content...)
+               }
+       } else {
+               detectContent = content
+       }
+       result, err := textDetector.DetectBest(detectContent)
        if err != nil {
                return "", err
        }

diff --git a/modules/templates/helper.go b/modules/templates/helper.go
index d55c122df09995e71ea93f71ad16e3ddf32df6f1..ce077d1a928e74f2bf26b3d2dc846f6c51476687 100644 (file)
--- a/modules/templates/helper.go
+++ b/modules/templates/helper.go
@@ -1,3 +1,4 @@
+// Copyright 2018 The Gitea Authors. All rights reserved.
 // Copyright 2014 The Gogs Authors. All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
@@ -275,7 +276,7 @@ func ToUTF8WithErr(content []byte) (string, error) {
        }
 
        // If there is an error, we concatenate the nicely decoded part and the
-       // original left over. This way we won't loose data.
+       // original left over. This way we won't lose data.
        result, n, err := transform.String(encoding.NewDecoder(), string(content))
        if err != nil {
                result = result + string(content[n:])
@@ -284,6 +285,28 @@ func ToUTF8WithErr(content []byte) (string, error) {
        return result, err
 }
 
+// ToUTF8WithFallback detects the encoding of content and coverts to UTF-8 if possible
+func ToUTF8WithFallback(content []byte) []byte {
+       charsetLabel, err := base.DetectEncoding(content)
+       if err != nil || charsetLabel == "UTF-8" {
+               return content
+       }
+
+       encoding, _ := charset.Lookup(charsetLabel)
+       if encoding == nil {
+               return content
+       }
+
+       // If there is an error, we concatenate the nicely decoded part and the
+       // original left over. This way we won't lose data.
+       result, n, err := transform.Bytes(encoding.NewDecoder(), content)
+       if err != nil {
+               return append(result, content[n:]...)
+       }
+
+       return result
+}
+
 // ToUTF8 converts content to UTF8 encoding and ignore error
 func ToUTF8(content string) string {
        res, _ := ToUTF8WithErr([]byte(content))

diff --git a/routers/repo/view.go b/routers/repo/view.go
index ff5c1afb49590e3e39e439f0aeb476ae8ce68ed3..210eb9fe5ffb495aa434df16eaa779a7a6b7a724 100644 (file)
--- a/routers/repo/view.go
+++ b/routers/repo/view.go
@@ -25,6 +25,7 @@ import (
        "code.gitea.io/gitea/modules/markup"
        "code.gitea.io/gitea/modules/setting"
        "code.gitea.io/gitea/modules/templates"
+
        "github.com/Unknwon/paginater"
 )
 
@@ -99,7 +100,8 @@ func renderDirectory(ctx *context.Context, treeLink string) {
                                ctx.Data["FileSize"] = readmeFile.Size()
                        } else {
                                d, _ := ioutil.ReadAll(dataRc)
-                               buf = append(buf, d...)
+                               buf = templates.ToUTF8WithFallback(append(buf, d...))
+
                                if markup.Type(readmeFile.Name()) != "" {
                                        ctx.Data["IsMarkup"] = true
                                        ctx.Data["FileContent"] = string(markup.Render(readmeFile.Name(), buf, treeLink, ctx.Repo.Repository.ComposeMetas()))
@@ -203,7 +205,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
                }
 
                d, _ := ioutil.ReadAll(dataRc)
-               buf = append(buf, d...)
+               buf = templates.ToUTF8WithFallback(append(buf, d...))
 
                readmeExist := markup.IsReadmeFile(blob.Name())
                ctx.Data["ReadmeExist"] = readmeExist