Memory usage improvements (#3013) (#3028)

* govendor update code.gitea.io/git

Signed-off-by: Duncan Ogilvie <mr.exodia.tpodt@gmail.com>
* Greatly improve memory usage

Signed-off-by: Duncan Ogilvie <mr.exodia.tpodt@gmail.com>

diff --git a/modules/context/repo.go b/modules/context/repo.go
index 38baa4230615148c6dbd4508be1a783df0709d3c..e8d084c342a164d163e1213ef7d28050af860de9 100644 (file)
--- a/modules/context/repo.go
+++ b/modules/context/repo.go
@@ -143,6 +143,9 @@ func (r *Repository) GetEditorconfig() (*editorconfig.Editorconfig, error) {
        if err != nil {
                return nil, err
        }
+       if treeEntry.Blob().Size() >= setting.UI.MaxDisplayFileSize {
+               return nil, git.ErrNotExist{ID: "", RelPath: ".editorconfig"}
+       }
        reader, err := treeEntry.Blob().Data()
        if err != nil {
                return nil, err

diff --git a/routers/repo/download.go b/routers/repo/download.go
index 78c60886078bcda434602390f3447772f98f9f0e..78c4b519bec2061db6163c8fffd4148a7feb5d1d 100644 (file)
--- a/routers/repo/download.go
+++ b/routers/repo/download.go
@@ -45,10 +45,11 @@ func ServeData(ctx *context.Context, name string, reader io.Reader) error {
 
 // ServeBlob download a git.Blob
 func ServeBlob(ctx *context.Context, blob *git.Blob) error {
-       dataRc, err := blob.Data()
+       dataRc, err := blob.DataAsync()
        if err != nil {
                return err
        }
+       defer dataRc.Close()
 
        return ServeData(ctx, ctx.Repo.TreePath, dataRc)
 }

diff --git a/routers/repo/editor.go b/routers/repo/editor.go
index a6cc9223647373468875eebc20bce8c49935219b..82b04a84d2053bbb98d0917372b2b708965fe1f4 100644 (file)
--- a/routers/repo/editor.go
+++ b/routers/repo/editor.go
@@ -73,11 +73,16 @@ func editFile(ctx *context.Context, isNewFile bool) {
 
                // No way to edit a directory online.
                if entry.IsDir() {
-                       ctx.Handle(404, "", nil)
+                       ctx.Handle(404, "entry.IsDir", nil)
                        return
                }
 
                blob := entry.Blob()
+               if blob.Size() >= setting.UI.MaxDisplayFileSize {
+                       ctx.Handle(404, "blob.Size", err)
+                       return
+               }
+
                dataRc, err := blob.Data()
                if err != nil {
                        ctx.Handle(404, "blob.Data", err)
@@ -93,7 +98,7 @@ func editFile(ctx *context.Context, isNewFile bool) {
 
                // Only text file are editable online.
                if !base.IsTextFile(buf) {
-                       ctx.Handle(404, "", nil)
+                       ctx.Handle(404, "base.IsTextFile", nil)
                        return
                }
 

diff --git a/routers/repo/issue.go b/routers/repo/issue.go
index c24a4e4360d415759ecd60d79630ec6f25f5059b..b45d521e5b1df3f092a2f9c3344737fb93320d4e 100644 (file)
--- a/routers/repo/issue.go
+++ b/routers/repo/issue.go
@@ -319,6 +319,9 @@ func getFileContentFromDefaultBranch(ctx *context.Context, filename string) (str
        if err != nil {
                return "", false
        }
+       if entry.Blob().Size() >= setting.UI.MaxDisplayFileSize {
+               return "", false
+       }
        r, err = entry.Blob().Data()
        if err != nil {
                return "", false

diff --git a/routers/repo/view.go b/routers/repo/view.go
index d43b4d7f78b7160e488d80e5fc8bf874a272bc71..a02acb0d6c13738e33afb0aeaeb07e68523cd3f3 100644 (file)
--- a/routers/repo/view.go
+++ b/routers/repo/view.go
@@ -76,11 +76,12 @@ func renderDirectory(ctx *context.Context, treeLink string) {
                ctx.Data["ReadmeInList"] = true
                ctx.Data["ReadmeExist"] = true
 
-               dataRc, err := readmeFile.Data()
+               dataRc, err := readmeFile.DataAsync()
                if err != nil {
                        ctx.Handle(500, "Data", err)
                        return
                }
+               defer dataRc.Close()
 
                buf := make([]byte, 1024)
                n, _ := dataRc.Read(buf)
@@ -91,14 +92,21 @@ func renderDirectory(ctx *context.Context, treeLink string) {
                ctx.Data["FileName"] = readmeFile.Name()
                // FIXME: what happens when README file is an image?
                if isTextFile {
-                       d, _ := ioutil.ReadAll(dataRc)
-                       buf = append(buf, d...)
-                       if markup.Type(readmeFile.Name()) != "" {
-                               ctx.Data["IsMarkup"] = true
-                               ctx.Data["FileContent"] = string(markup.Render(readmeFile.Name(), buf, treeLink, ctx.Repo.Repository.ComposeMetas()))
+                       if readmeFile.Size() >= setting.UI.MaxDisplayFileSize {
+                               // Pretend that this is a normal text file to display 'This file is too large to be shown'
+                               ctx.Data["IsFileTooLarge"] = true
+                               ctx.Data["IsTextFile"] = true
+                               ctx.Data["FileSize"] = readmeFile.Size()
                        } else {
-                               ctx.Data["IsRenderedHTML"] = true
-                               ctx.Data["FileContent"] = string(bytes.Replace(buf, []byte("\n"), []byte(`<br>`), -1))
+                               d, _ := ioutil.ReadAll(dataRc)
+                               buf = append(buf, d...)
+                               if markup.Type(readmeFile.Name()) != "" {
+                                       ctx.Data["IsMarkup"] = true
+                                       ctx.Data["FileContent"] = string(markup.Render(readmeFile.Name(), buf, treeLink, ctx.Repo.Repository.ComposeMetas()))
+                               } else {
+                                       ctx.Data["IsRenderedHTML"] = true
+                                       ctx.Data["FileContent"] = string(bytes.Replace(buf, []byte("\n"), []byte(`<br>`), -1))
+                               }
                        }
                }
        }
@@ -135,11 +143,12 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
        ctx.Data["IsViewFile"] = true
 
        blob := entry.Blob()
-       dataRc, err := blob.Data()
+       dataRc, err := blob.DataAsync()
        if err != nil {
-               ctx.Handle(500, "Data", err)
+               ctx.Handle(500, "DataAsync", err)
                return
        }
+       defer dataRc.Close()
 
        ctx.Data["FileSize"] = blob.Size()
        ctx.Data["FileName"] = blob.Name()

diff --git a/vendor/code.gitea.io/git/blob.go b/vendor/code.gitea.io/git/blob.go
index 10b8ea4c9f166d2bf28272f3d4af3809fcd73f0f..a6e392eeb50a611d47fc68c2e2864a2519deb83d 100644 (file)
--- a/vendor/code.gitea.io/git/blob.go
+++ b/vendor/code.gitea.io/git/blob.go
@@ -6,7 +6,11 @@ package git
 
 import (
        "bytes"
+       "fmt"
        "io"
+       "io/ioutil"
+       "os"
+       "os/exec"
 )
 
 // Blob represents a Git object.
@@ -18,14 +22,52 @@ type Blob struct {
 // Data gets content of blob all at once and wrap it as io.Reader.
 // This can be very slow and memory consuming for huge content.
 func (b *Blob) Data() (io.Reader, error) {
-       stdout, err := NewCommand("show", b.ID.String()).RunInDirBytes(b.repo.Path)
-       if err != nil {
-               return nil, err
+       stdout := new(bytes.Buffer)
+       stderr := new(bytes.Buffer)
+
+       // Preallocate memory to save ~50% memory usage on big files.
+       stdout.Grow(int(b.Size() + 2048))
+
+       if err := b.DataPipeline(stdout, stderr); err != nil {
+               return nil, concatenateError(err, stderr.String())
        }
-       return bytes.NewBuffer(stdout), nil
+       return stdout, nil
 }
 
 // DataPipeline gets content of blob and write the result or error to stdout or stderr
 func (b *Blob) DataPipeline(stdout, stderr io.Writer) error {
        return NewCommand("show", b.ID.String()).RunInDirPipeline(b.repo.Path, stdout, stderr)
 }
+
+type cmdReadCloser struct {
+       cmd    *exec.Cmd
+       stdout io.Reader
+}
+
+func (c cmdReadCloser) Read(p []byte) (int, error) {
+       return c.stdout.Read(p)
+}
+
+func (c cmdReadCloser) Close() error {
+       io.Copy(ioutil.Discard, c.stdout)
+       return c.cmd.Wait()
+}
+
+// DataAsync gets a ReadCloser for the contents of a blob without reading it all.
+// Calling the Close function on the result will discard all unread output.
+func (b *Blob) DataAsync() (io.ReadCloser, error) {
+       cmd := exec.Command("git", "show", b.ID.String())
+       cmd.Dir = b.repo.Path
+       cmd.Stderr = os.Stderr
+
+       stdout, err := cmd.StdoutPipe()
+       if err != nil {
+               return nil, fmt.Errorf("StdoutPipe: %v", err)
+       }
+
+       if err = cmd.Start(); err != nil {
+               return nil, fmt.Errorf("Start: %v", err)
+       }
+
+       return cmdReadCloser{stdout: stdout, cmd: cmd}, nil
+}

diff --git a/vendor/code.gitea.io/git/commit.go b/vendor/code.gitea.io/git/commit.go
index c2954123816e3732ecb18c70935cb000568e23a6..299a2381b65b6ab0fb70489f5af150169f842f50 100644 (file)
--- a/vendor/code.gitea.io/git/commit.go
+++ b/vendor/code.gitea.io/git/commit.go
@@ -98,10 +98,11 @@ func (c *Commit) IsImageFile(name string) bool {
                return false
        }
 
-       dataRc, err := blob.Data()
+       dataRc, err := blob.DataAsync()
        if err != nil {
                return false
        }
+       defer dataRc.Close()
        buf := make([]byte, 1024)
        n, _ := dataRc.Read(buf)
        buf = buf[:n]

diff --git a/vendor/code.gitea.io/git/git.go b/vendor/code.gitea.io/git/git.go
index 9ec20c97e13ac5c713a1fddab952ba6657290287..150b80fb076b840a9ca1e40f0fa14e78f54d4190 100644 (file)
--- a/vendor/code.gitea.io/git/git.go
+++ b/vendor/code.gitea.io/git/git.go
@@ -25,7 +25,7 @@ var (
        // Prefix the log prefix
        Prefix = "[git-module] "
        // GitVersionRequired is the minimum Git version required
-       GitVersionRequired = "1.8.1.6"
+       GitVersionRequired = "1.7.2"
 )
 
 func log(format string, args ...interface{}) {

diff --git a/vendor/vendor.json b/vendor/vendor.json
index b5bc882ecf7fdff0b944868f0ae5d9b3e77f3a85..4b997c671170dffc177cacb0e51270314a23f715 100644 (file)
--- a/vendor/vendor.json
+++ b/vendor/vendor.json
@@ -3,10 +3,10 @@
        "ignore": "test appengine",
        "package": [
                {
-                       "checksumSHA1": "JN/re4+x/hCzMLGHmieUcykVDAg=",
+                       "checksumSHA1": "vAVjAz7Wpjnu7GGba4JLIDTpQEw=",
                        "path": "code.gitea.io/git",
-                       "revision": "d47b98c44c9a6472e44ab80efe65235e11c6da2a",
-                       "revisionTime": "2017-10-23T00:52:09Z"
+                       "revision": "f9dd6826bbb51c92c6964ce18176c304ea286e54",
+                       "revisionTime": "2017-11-28T15:25:05Z"
                },
                {
                        "checksumSHA1": "OICEgmUefW4L4l/FK/NVFnl/aOM=",