* govendor update code.gitea.io/git Signed-off-by: Duncan Ogilvie <mr.exodia.tpodt@gmail.com> * Greatly improve memory usage Signed-off-by: Duncan Ogilvie <mr.exodia.tpodt@gmail.com>tags/v1.3.0
@@ -143,6 +143,9 @@ func (r *Repository) GetEditorconfig() (*editorconfig.Editorconfig, error) { | |||
if err != nil { | |||
return nil, err | |||
} | |||
if treeEntry.Blob().Size() >= setting.UI.MaxDisplayFileSize { | |||
return nil, git.ErrNotExist{ID: "", RelPath: ".editorconfig"} | |||
} | |||
reader, err := treeEntry.Blob().Data() | |||
if err != nil { | |||
return nil, err |
@@ -45,10 +45,11 @@ func ServeData(ctx *context.Context, name string, reader io.Reader) error { | |||
// ServeBlob download a git.Blob | |||
func ServeBlob(ctx *context.Context, blob *git.Blob) error { | |||
dataRc, err := blob.Data() | |||
dataRc, err := blob.DataAsync() | |||
if err != nil { | |||
return err | |||
} | |||
defer dataRc.Close() | |||
return ServeData(ctx, ctx.Repo.TreePath, dataRc) | |||
} |
@@ -73,11 +73,16 @@ func editFile(ctx *context.Context, isNewFile bool) { | |||
// No way to edit a directory online. | |||
if entry.IsDir() { | |||
ctx.Handle(404, "", nil) | |||
ctx.Handle(404, "entry.IsDir", nil) | |||
return | |||
} | |||
blob := entry.Blob() | |||
if blob.Size() >= setting.UI.MaxDisplayFileSize { | |||
ctx.Handle(404, "blob.Size", err) | |||
return | |||
} | |||
dataRc, err := blob.Data() | |||
if err != nil { | |||
ctx.Handle(404, "blob.Data", err) | |||
@@ -93,7 +98,7 @@ func editFile(ctx *context.Context, isNewFile bool) { | |||
// Only text file are editable online. | |||
if !base.IsTextFile(buf) { | |||
ctx.Handle(404, "", nil) | |||
ctx.Handle(404, "base.IsTextFile", nil) | |||
return | |||
} | |||
@@ -319,6 +319,9 @@ func getFileContentFromDefaultBranch(ctx *context.Context, filename string) (str | |||
if err != nil { | |||
return "", false | |||
} | |||
if entry.Blob().Size() >= setting.UI.MaxDisplayFileSize { | |||
return "", false | |||
} | |||
r, err = entry.Blob().Data() | |||
if err != nil { | |||
return "", false |
@@ -76,11 +76,12 @@ func renderDirectory(ctx *context.Context, treeLink string) { | |||
ctx.Data["ReadmeInList"] = true | |||
ctx.Data["ReadmeExist"] = true | |||
dataRc, err := readmeFile.Data() | |||
dataRc, err := readmeFile.DataAsync() | |||
if err != nil { | |||
ctx.Handle(500, "Data", err) | |||
return | |||
} | |||
defer dataRc.Close() | |||
buf := make([]byte, 1024) | |||
n, _ := dataRc.Read(buf) | |||
@@ -91,14 +92,21 @@ func renderDirectory(ctx *context.Context, treeLink string) { | |||
ctx.Data["FileName"] = readmeFile.Name() | |||
// FIXME: what happens when README file is an image? | |||
if isTextFile { | |||
d, _ := ioutil.ReadAll(dataRc) | |||
buf = append(buf, d...) | |||
if markup.Type(readmeFile.Name()) != "" { | |||
ctx.Data["IsMarkup"] = true | |||
ctx.Data["FileContent"] = string(markup.Render(readmeFile.Name(), buf, treeLink, ctx.Repo.Repository.ComposeMetas())) | |||
if readmeFile.Size() >= setting.UI.MaxDisplayFileSize { | |||
// Pretend that this is a normal text file to display 'This file is too large to be shown' | |||
ctx.Data["IsFileTooLarge"] = true | |||
ctx.Data["IsTextFile"] = true | |||
ctx.Data["FileSize"] = readmeFile.Size() | |||
} else { | |||
ctx.Data["IsRenderedHTML"] = true | |||
ctx.Data["FileContent"] = string(bytes.Replace(buf, []byte("\n"), []byte(`<br>`), -1)) | |||
d, _ := ioutil.ReadAll(dataRc) | |||
buf = append(buf, d...) | |||
if markup.Type(readmeFile.Name()) != "" { | |||
ctx.Data["IsMarkup"] = true | |||
ctx.Data["FileContent"] = string(markup.Render(readmeFile.Name(), buf, treeLink, ctx.Repo.Repository.ComposeMetas())) | |||
} else { | |||
ctx.Data["IsRenderedHTML"] = true | |||
ctx.Data["FileContent"] = string(bytes.Replace(buf, []byte("\n"), []byte(`<br>`), -1)) | |||
} | |||
} | |||
} | |||
} | |||
@@ -135,11 +143,12 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st | |||
ctx.Data["IsViewFile"] = true | |||
blob := entry.Blob() | |||
dataRc, err := blob.Data() | |||
dataRc, err := blob.DataAsync() | |||
if err != nil { | |||
ctx.Handle(500, "Data", err) | |||
ctx.Handle(500, "DataAsync", err) | |||
return | |||
} | |||
defer dataRc.Close() | |||
ctx.Data["FileSize"] = blob.Size() | |||
ctx.Data["FileName"] = blob.Name() |
@@ -6,7 +6,11 @@ package git | |||
import ( | |||
"bytes" | |||
"fmt" | |||
"io" | |||
"io/ioutil" | |||
"os" | |||
"os/exec" | |||
) | |||
// Blob represents a Git object. | |||
@@ -18,14 +22,52 @@ type Blob struct { | |||
// Data gets content of blob all at once and wrap it as io.Reader. | |||
// This can be very slow and memory consuming for huge content. | |||
func (b *Blob) Data() (io.Reader, error) { | |||
stdout, err := NewCommand("show", b.ID.String()).RunInDirBytes(b.repo.Path) | |||
if err != nil { | |||
return nil, err | |||
stdout := new(bytes.Buffer) | |||
stderr := new(bytes.Buffer) | |||
// Preallocate memory to save ~50% memory usage on big files. | |||
stdout.Grow(int(b.Size() + 2048)) | |||
if err := b.DataPipeline(stdout, stderr); err != nil { | |||
return nil, concatenateError(err, stderr.String()) | |||
} | |||
return bytes.NewBuffer(stdout), nil | |||
return stdout, nil | |||
} | |||
// DataPipeline gets content of blob and write the result or error to stdout or stderr | |||
func (b *Blob) DataPipeline(stdout, stderr io.Writer) error { | |||
return NewCommand("show", b.ID.String()).RunInDirPipeline(b.repo.Path, stdout, stderr) | |||
} | |||
type cmdReadCloser struct { | |||
cmd *exec.Cmd | |||
stdout io.Reader | |||
} | |||
func (c cmdReadCloser) Read(p []byte) (int, error) { | |||
return c.stdout.Read(p) | |||
} | |||
func (c cmdReadCloser) Close() error { | |||
io.Copy(ioutil.Discard, c.stdout) | |||
return c.cmd.Wait() | |||
} | |||
// DataAsync gets a ReadCloser for the contents of a blob without reading it all. | |||
// Calling the Close function on the result will discard all unread output. | |||
func (b *Blob) DataAsync() (io.ReadCloser, error) { | |||
cmd := exec.Command("git", "show", b.ID.String()) | |||
cmd.Dir = b.repo.Path | |||
cmd.Stderr = os.Stderr | |||
stdout, err := cmd.StdoutPipe() | |||
if err != nil { | |||
return nil, fmt.Errorf("StdoutPipe: %v", err) | |||
} | |||
if err = cmd.Start(); err != nil { | |||
return nil, fmt.Errorf("Start: %v", err) | |||
} | |||
return cmdReadCloser{stdout: stdout, cmd: cmd}, nil | |||
} |
@@ -98,10 +98,11 @@ func (c *Commit) IsImageFile(name string) bool { | |||
return false | |||
} | |||
dataRc, err := blob.Data() | |||
dataRc, err := blob.DataAsync() | |||
if err != nil { | |||
return false | |||
} | |||
defer dataRc.Close() | |||
buf := make([]byte, 1024) | |||
n, _ := dataRc.Read(buf) | |||
buf = buf[:n] |
@@ -25,7 +25,7 @@ var ( | |||
// Prefix the log prefix | |||
Prefix = "[git-module] " | |||
// GitVersionRequired is the minimum Git version required | |||
GitVersionRequired = "1.8.1.6" | |||
GitVersionRequired = "1.7.2" | |||
) | |||
func log(format string, args ...interface{}) { |
@@ -3,10 +3,10 @@ | |||
"ignore": "test appengine", | |||
"package": [ | |||
{ | |||
"checksumSHA1": "JN/re4+x/hCzMLGHmieUcykVDAg=", | |||
"checksumSHA1": "vAVjAz7Wpjnu7GGba4JLIDTpQEw=", | |||
"path": "code.gitea.io/git", | |||
"revision": "d47b98c44c9a6472e44ab80efe65235e11c6da2a", | |||
"revisionTime": "2017-10-23T00:52:09Z" | |||
"revision": "f9dd6826bbb51c92c6964ce18176c304ea286e54", | |||
"revisionTime": "2017-11-28T15:25:05Z" | |||
}, | |||
{ | |||
"checksumSHA1": "OICEgmUefW4L4l/FK/NVFnl/aOM=", |