toutf8 improved & add max git diff lines

diff --git a/conf/app.ini b/conf/app.ini
index be49e06411d3aa4410f4e83f0a62172ca45dbfef..cb907b22de2b5b301466592c46eb76b60a6a701f 100644 (file)
--- a/conf/app.ini
+++ b/conf/app.ini
@@ -255,3 +255,6 @@ CONN =
 [i18n]
 LANGS = en-US,zh-CN,de-DE
 NAMES = English,简体中文,Deutsch
+
+[git]
+MAX_GITDIFF_LINES = 10000
\ No newline at end of file

diff --git a/models/git_diff.go b/models/git_diff.go
index bf7a9cd575b145b0c87809ca809ce8de48c22e13..e093e7ab1bb5558e164945423cad248f4bec183c 100644 (file)
--- a/models/git_diff.go
+++ b/models/git_diff.go
@@ -70,7 +70,7 @@ func (diff *Diff) NumFiles() int {
 
 const DIFF_HEAD = "diff --git "
 
-func ParsePatch(pid int64, cmd *exec.Cmd, reader io.Reader) (*Diff, error) {
+func ParsePatch(pid int64, maxlines int, cmd *exec.Cmd, reader io.Reader) (*Diff, error) {
        scanner := bufio.NewScanner(reader)
        var (
                curFile    *DiffFile
@@ -79,6 +79,7 @@ func ParsePatch(pid int64, cmd *exec.Cmd, reader io.Reader) (*Diff, error) {
                }
 
                leftLine, rightLine int
+               isTooLong           bool
        )
 
        diff := &Diff{Files: make([]*DiffFile, 0)}
@@ -90,16 +91,17 @@ func ParsePatch(pid int64, cmd *exec.Cmd, reader io.Reader) (*Diff, error) {
                        continue
                }
 
+               if line == "" {
+                       continue
+               }
+
                i = i + 1
 
-               // Diff data too large.
-               if i == 5000 {
+               // Diff data too large, we only show the first about maxlines lines
+               if i == maxlines {
+                       isTooLong = true
                        log.Warn("Diff data too large")
-                       return &Diff{}, nil
-               }
-
-               if line == "" {
-                       continue
+                       //return &Diff{}, nil
                }
 
                switch {
@@ -110,6 +112,10 @@ func ParsePatch(pid int64, cmd *exec.Cmd, reader io.Reader) (*Diff, error) {
                        curSection.Lines = append(curSection.Lines, diffLine)
                        continue
                case line[0] == '@':
+                       if isTooLong {
+                               return diff, nil
+                       }
+
                        curSection = &DiffSection{}
                        curFile.Sections = append(curFile.Sections, curSection)
                        ss := strings.Split(line, "@@")
@@ -143,6 +149,10 @@ func ParsePatch(pid int64, cmd *exec.Cmd, reader io.Reader) (*Diff, error) {
 
                // Get new file.
                if strings.HasPrefix(line, DIFF_HEAD) {
+                       if isTooLong {
+                               return diff, nil
+                       }
+
                        fs := strings.Split(line[len(DIFF_HEAD):], " ")
                        a := fs[0]
 
@@ -174,7 +184,7 @@ func ParsePatch(pid int64, cmd *exec.Cmd, reader io.Reader) (*Diff, error) {
        return diff, nil
 }
 
-func GetDiffRange(repoPath, beforeCommitId string, afterCommitId string) (*Diff, error) {
+func GetDiffRange(repoPath, beforeCommitId string, afterCommitId string, maxlines int) (*Diff, error) {
        repo, err := git.OpenRepository(repoPath)
        if err != nil {
                return nil, err
@@ -228,9 +238,9 @@ func GetDiffRange(repoPath, beforeCommitId string, afterCommitId string) (*Diff,
                }
        }()
 
-       return ParsePatch(pid, cmd, rd)
+       return ParsePatch(pid, maxlines, cmd, rd)
 }
 
-func GetDiffCommit(repoPath, commitId string) (*Diff, error) {
-       return GetDiffRange(repoPath, "", commitId)
+func GetDiffCommit(repoPath, commitId string, maxlines int) (*Diff, error) {
+       return GetDiffRange(repoPath, "", commitId, maxlines)
 }

diff --git a/modules/base/template.go b/modules/base/template.go
index f2ae00b9159c0200e6dafed31d771b945e2eea08..6419572912a657a7fab60d8f1b6302de64f3a31f 100644 (file)
--- a/modules/base/template.go
+++ b/modules/base/template.go
@@ -8,13 +8,16 @@ import (
        "bytes"
        "container/list"
        "encoding/json"
+       "errors"
        "fmt"
        "html/template"
        "runtime"
        "strings"
        "time"
+       "code.google.com/p/mahonia"
 
        "github.com/gogits/gogs/modules/setting"
+       "github.com/saintfish/chardet"
 )
 
 func Str2html(raw string) template.HTML {
@@ -45,6 +48,29 @@ func ShortSha(sha1 string) string {
        return sha1
 }
 
+func ToUtf8WithErr(content []byte) (error, string) {
+       detector := chardet.NewTextDetector()
+       result, err := detector.DetectBest(content)
+       if err != nil {
+               return err, ""
+       }
+
+       if result.Charset == "utf8" {
+               return nil, string(content)
+       }
+
+       decoder := mahonia.NewDecoder(result.Charset)
+       if decoder != nil {
+               return nil, decoder.ConvertString(string(content))
+       }
+       return errors.New("unknow char decoder"), string(content)
+}
+
+func ToUtf8(content string) string {
+       _, res := ToUtf8WithErr([]byte(content))
+       return res
+}
+
 var mailDomains = map[string]string{
        "gmail.com": "gmail.com",
 }
@@ -103,6 +129,7 @@ var TemplateFuncs template.FuncMap = map[string]interface{}{
        "ActionContent2Commits": ActionContent2Commits,
        "Oauth2Icon":            Oauth2Icon,
        "Oauth2Name":            Oauth2Name,
+       "ToUtf8":                ToUtf8,
 }
 
 type Actioner interface {

diff --git a/modules/setting/setting.go b/modules/setting/setting.go
index 199b4f2c278f803657423abc572cfae60ad3c1cd..011c00c08c8ca74cff6f79c3bbc49d3a1cf90bc3 100644 (file)
--- a/modules/setting/setting.go
+++ b/modules/setting/setting.go
@@ -64,6 +64,7 @@ var (
        // Picture settings.
        PictureService  string
        DisableGravatar bool
+       MaxGitDiffLines int
 
        // Log settings.
        LogRootPath string
@@ -241,6 +242,8 @@ func NewConfigContext() {
                []string{"server"})
        DisableGravatar = Cfg.MustBool("picture", "DISABLE_GRAVATAR")
 
+       MaxGitDiffLines = Cfg.MustInt("git", "MAX_GITDIFF_LINES", 5000)
+
        Langs = Cfg.MustValueArray("i18n", "LANGS", ",")
        Names = Cfg.MustValueArray("i18n", "NAMES", ",")
 }

diff --git a/routers/repo/commit.go b/routers/repo/commit.go
index 54acc85b3e2354afe09a3e9a5f569a4063c5b91f..f7feb4d95ec9a408fd13a91c25eae7777fb98a4c 100644 (file)
--- a/routers/repo/commit.go
+++ b/routers/repo/commit.go
@@ -12,6 +12,7 @@ import (
        "github.com/gogits/gogs/models"
        "github.com/gogits/gogs/modules/base"
        "github.com/gogits/gogs/modules/middleware"
+       "github.com/gogits/gogs/modules/setting"
 )
 
 const (
@@ -114,7 +115,8 @@ func Diff(ctx *middleware.Context) {
 
        commit := ctx.Repo.Commit
 
-       diff, err := models.GetDiffCommit(models.RepoPath(userName, repoName), commitId)
+       diff, err := models.GetDiffCommit(models.RepoPath(userName, repoName),
+               commitId, setting.MaxGitDiffLines)
        if err != nil {
                ctx.Handle(404, "GetDiffCommit", err)
                return
@@ -176,7 +178,8 @@ func CompareDiff(ctx *middleware.Context) {
                return
        }
 
-       diff, err := models.GetDiffRange(models.RepoPath(userName, repoName), beforeCommitId, afterCommitId)
+       diff, err := models.GetDiffRange(models.RepoPath(userName, repoName), beforeCommitId,
+               afterCommitId, setting.MaxGitDiffLines)
        if err != nil {
                ctx.Handle(404, "GetDiffRange", err)
                return

diff --git a/routers/repo/view.go b/routers/repo/view.go
index f98eee037df20ddc84340dcb76f474414932b232..e42894ae7351075f083049c21ba6f68be6270a37 100644 (file)
--- a/routers/repo/view.go
+++ b/routers/repo/view.go
@@ -11,12 +11,9 @@ import (
        "path/filepath"
        "strings"
 
-       "github.com/saintfish/chardet"
-
        "github.com/gogits/gogs/modules/base"
        "github.com/gogits/gogs/modules/git"
        "github.com/gogits/gogs/modules/log"
-       "github.com/gogits/gogs/modules/mahonia"
        "github.com/gogits/gogs/modules/middleware"
 )
 
@@ -24,21 +21,6 @@ const (
        HOME base.TplName = "repo/home"
 )
 
-func toUtf8(content []byte) (error, string) {
-       detector := chardet.NewTextDetector()
-       result, err := detector.DetectBest(content)
-       if err != nil {
-               return err, ""
-       }
-
-       if result.Charset == "utf8" {
-               return nil, string(content)
-       }
-
-       decoder := mahonia.NewDecoder(result.Charset)
-       return nil, decoder.ConvertString(string(content))
-}
-
 func Home(ctx *middleware.Context) {
        ctx.Data["Title"] = ctx.Repo.Repository.Name
 
@@ -117,7 +99,7 @@ func Home(ctx *middleware.Context) {
                                if readmeExist {
                                        ctx.Data["FileContent"] = string(base.RenderMarkdown(buf, ""))
                                } else {
-                                       if err, content := toUtf8(buf); err != nil {
+                                       if err, content := base.ToUtf8WithErr(buf); err != nil {
                                                if err != nil {
                                                        log.Error(4, "Convert content encoding: %s", err)
                                                }

diff --git a/templates/repo/diff.tmpl b/templates/repo/diff.tmpl
index 787334508362f7c0bb09f9cdadf41a38c1485620..a2150f28498798d134fb7f4bfff3a147cdfa0ea6 100644 (file)
--- a/templates/repo/diff.tmpl
+++ b/templates/repo/diff.tmpl
@@ -105,7 +105,7 @@
                                 <span rel="L1">{{if .RightIdx}}{{.RightIdx}}{{end}}</span>
                             </td>
                             <td class="lines-code">
-                                <pre>{{.Content}}</pre>
+                                <pre>{{ToUtf8 .Content}}</pre>
                             </td>
                         </tr>
                         {{end}}