Use common git cat-file --batch and git cat-file --batch-check to significantly reduce calls to git. Signed-off-by: Andrew Thornton <art27@cantab.net>tags/v1.15.0-rc1
@@ -905,12 +905,18 @@ func (ctx *Context) IssueTemplatesFromDefaultBranch() []api.IssueTemplate { | |||
log.Debug("DataAsync: %v", err) | |||
continue | |||
} | |||
defer r.Close() | |||
closed := false | |||
defer func() { | |||
if !closed { | |||
_ = r.Close() | |||
} | |||
}() | |||
data, err := ioutil.ReadAll(r) | |||
if err != nil { | |||
log.Debug("ReadAll: %v", err) | |||
continue | |||
} | |||
_ = r.Close() | |||
var it api.IssueTemplate | |||
content, err := markdown.ExtractMetadata(string(data), &it) | |||
if err != nil { |
@@ -13,9 +13,44 @@ import ( | |||
"strings" | |||
) | |||
// WriteCloserError wraps an io.WriteCloser with an additional CloseWithError function | |||
type WriteCloserError interface { | |||
io.WriteCloser | |||
CloseWithError(err error) error | |||
} | |||
// CatFileBatchCheck opens git cat-file --batch-check in the provided repo and returns a stdin pipe, a stdout reader and cancel function | |||
func CatFileBatchCheck(repoPath string) (WriteCloserError, *bufio.Reader, func()) { | |||
batchStdinReader, batchStdinWriter := io.Pipe() | |||
batchStdoutReader, batchStdoutWriter := io.Pipe() | |||
cancel := func() { | |||
_ = batchStdinReader.Close() | |||
_ = batchStdinWriter.Close() | |||
_ = batchStdoutReader.Close() | |||
_ = batchStdoutWriter.Close() | |||
} | |||
go func() { | |||
stderr := strings.Builder{} | |||
err := NewCommand("cat-file", "--batch-check").RunInDirFullPipeline(repoPath, batchStdoutWriter, &stderr, batchStdinReader) | |||
if err != nil { | |||
_ = batchStdoutWriter.CloseWithError(ConcatenateError(err, (&stderr).String())) | |||
_ = batchStdinReader.CloseWithError(ConcatenateError(err, (&stderr).String())) | |||
} else { | |||
_ = batchStdoutWriter.Close() | |||
_ = batchStdinReader.Close() | |||
} | |||
}() | |||
// For simplicities sake we'll us a buffered reader to read from the cat-file --batch | |||
batchReader := bufio.NewReader(batchStdoutReader) | |||
return batchStdinWriter, batchReader, cancel | |||
} | |||
// CatFileBatch opens git cat-file --batch in the provided repo and returns a stdin pipe, a stdout reader and cancel function | |||
func CatFileBatch(repoPath string) (*io.PipeWriter, *bufio.Reader, func()) { | |||
// Next feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary. | |||
func CatFileBatch(repoPath string) (WriteCloserError, *bufio.Reader, func()) { | |||
// We often want to feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary. | |||
// so let's create a batch stdin and stdout | |||
batchStdinReader, batchStdinWriter := io.Pipe() | |||
batchStdoutReader, batchStdoutWriter := io.Pipe() | |||
@@ -47,6 +82,7 @@ func CatFileBatch(repoPath string) (*io.PipeWriter, *bufio.Reader, func()) { | |||
// ReadBatchLine reads the header line from cat-file --batch | |||
// We expect: | |||
// <sha> SP <type> SP <size> LF | |||
// sha is a 40byte not 20byte here | |||
func ReadBatchLine(rd *bufio.Reader) (sha []byte, typ string, size int64, err error) { | |||
sha, err = rd.ReadBytes(' ') | |||
if err != nil { | |||
@@ -54,19 +90,20 @@ func ReadBatchLine(rd *bufio.Reader) (sha []byte, typ string, size int64, err er | |||
} | |||
sha = sha[:len(sha)-1] | |||
typ, err = rd.ReadString(' ') | |||
typ, err = rd.ReadString('\n') | |||
if err != nil { | |||
return | |||
} | |||
typ = typ[:len(typ)-1] | |||
var sizeStr string | |||
sizeStr, err = rd.ReadString('\n') | |||
if err != nil { | |||
idx := strings.Index(typ, " ") | |||
if idx < 0 { | |||
err = ErrNotExist{ID: string(sha)} | |||
return | |||
} | |||
sizeStr := typ[idx+1 : len(typ)-1] | |||
typ = typ[:idx] | |||
size, err = strconv.ParseInt(sizeStr[:len(sizeStr)-1], 10, 64) | |||
size, err = strconv.ParseInt(sizeStr, 10, 64) | |||
return | |||
} | |||
@@ -128,7 +165,7 @@ headerLoop: | |||
} | |||
// Discard the rest of the commit | |||
discard := size - n | |||
discard := size - n + 1 | |||
for discard > math.MaxInt32 { | |||
_, err := rd.Discard(math.MaxInt32) | |||
if err != nil { |
@@ -8,48 +8,54 @@ package git | |||
import ( | |||
"bufio" | |||
"bytes" | |||
"io" | |||
"strconv" | |||
"strings" | |||
"io/ioutil" | |||
"math" | |||
) | |||
// Blob represents a Git object. | |||
type Blob struct { | |||
ID SHA1 | |||
gotSize bool | |||
size int64 | |||
repoPath string | |||
name string | |||
gotSize bool | |||
size int64 | |||
name string | |||
repo *Repository | |||
} | |||
// DataAsync gets a ReadCloser for the contents of a blob without reading it all. | |||
// Calling the Close function on the result will discard all unread output. | |||
func (b *Blob) DataAsync() (io.ReadCloser, error) { | |||
stdoutReader, stdoutWriter := io.Pipe() | |||
wr, rd, cancel := b.repo.CatFileBatch() | |||
go func() { | |||
stderr := &strings.Builder{} | |||
err := NewCommand("cat-file", "--batch").RunInDirFullPipeline(b.repoPath, stdoutWriter, stderr, strings.NewReader(b.ID.String()+"\n")) | |||
if err != nil { | |||
err = ConcatenateError(err, stderr.String()) | |||
_ = stdoutWriter.CloseWithError(err) | |||
} else { | |||
_ = stdoutWriter.Close() | |||
} | |||
}() | |||
bufReader := bufio.NewReader(stdoutReader) | |||
_, _, size, err := ReadBatchLine(bufReader) | |||
_, err := wr.Write([]byte(b.ID.String() + "\n")) | |||
if err != nil { | |||
stdoutReader.Close() | |||
cancel() | |||
return nil, err | |||
} | |||
_, _, size, err := ReadBatchLine(rd) | |||
if err != nil { | |||
cancel() | |||
return nil, err | |||
} | |||
b.gotSize = true | |||
b.size = size | |||
return &LimitedReaderCloser{ | |||
R: bufReader, | |||
C: stdoutReader, | |||
N: size, | |||
if size < 4096 { | |||
bs, err := ioutil.ReadAll(io.LimitReader(rd, size)) | |||
if err != nil { | |||
cancel() | |||
return nil, err | |||
} | |||
_, err = rd.Discard(1) | |||
return ioutil.NopCloser(bytes.NewReader(bs)), err | |||
} | |||
return &blobReader{ | |||
rd: rd, | |||
n: size, | |||
cancel: cancel, | |||
}, nil | |||
} | |||
@@ -59,18 +65,66 @@ func (b *Blob) Size() int64 { | |||
return b.size | |||
} | |||
size, err := NewCommand("cat-file", "-s", b.ID.String()).RunInDir(b.repoPath) | |||
wr, rd, cancel := b.repo.CatFileBatchCheck() | |||
defer cancel() | |||
_, err := wr.Write([]byte(b.ID.String() + "\n")) | |||
if err != nil { | |||
log("error whilst reading size for %s in %s. Error: %v", b.ID.String(), b.repoPath, err) | |||
log("error whilst reading size for %s in %s. Error: %v", b.ID.String(), b.repo.Path, err) | |||
return 0 | |||
} | |||
b.size, err = strconv.ParseInt(size[:len(size)-1], 10, 64) | |||
_, _, b.size, err = ReadBatchLine(rd) | |||
if err != nil { | |||
log("error whilst parsing size %s for %s in %s. Error: %v", size, b.ID.String(), b.repoPath, err) | |||
log("error whilst reading size for %s in %s. Error: %v", b.ID.String(), b.repo.Path, err) | |||
return 0 | |||
} | |||
b.gotSize = true | |||
return b.size | |||
} | |||
type blobReader struct { | |||
rd *bufio.Reader | |||
n int64 | |||
cancel func() | |||
} | |||
func (b *blobReader) Read(p []byte) (n int, err error) { | |||
if b.n <= 0 { | |||
return 0, io.EOF | |||
} | |||
if int64(len(p)) > b.n { | |||
p = p[0:b.n] | |||
} | |||
n, err = b.rd.Read(p) | |||
b.n -= int64(n) | |||
return | |||
} | |||
// Close implements io.Closer | |||
func (b *blobReader) Close() error { | |||
if b.n > 0 { | |||
for b.n > math.MaxInt32 { | |||
n, err := b.rd.Discard(math.MaxInt32) | |||
b.n -= int64(n) | |||
if err != nil { | |||
b.cancel() | |||
return err | |||
} | |||
b.n -= math.MaxInt32 | |||
} | |||
n, err := b.rd.Discard(int(b.n)) | |||
b.n -= int64(n) | |||
if err != nil { | |||
b.cancel() | |||
return err | |||
} | |||
} | |||
if b.n == 0 { | |||
_, err := b.rd.Discard(1) | |||
b.n-- | |||
b.cancel() | |||
return err | |||
} | |||
return nil | |||
} |
@@ -29,9 +29,10 @@ func TestBlob_Data(t *testing.T) { | |||
r, err := testBlob.DataAsync() | |||
assert.NoError(t, err) | |||
require.NotNil(t, r) | |||
defer r.Close() | |||
data, err := ioutil.ReadAll(r) | |||
assert.NoError(t, r.Close()) | |||
assert.NoError(t, err) | |||
assert.Equal(t, output, string(data)) | |||
} | |||
@@ -54,7 +55,7 @@ func Benchmark_Blob_Data(b *testing.B) { | |||
if err != nil { | |||
b.Fatal(err) | |||
} | |||
defer r.Close() | |||
ioutil.ReadAll(r) | |||
_ = r.Close() | |||
} | |||
} |
@@ -102,7 +102,7 @@ func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache *LastCo | |||
} | |||
func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*Commit, []string, error) { | |||
wr, rd, cancel := CatFileBatch(cache.repo.Path) | |||
wr, rd, cancel := cache.repo.CatFileBatch() | |||
defer cancel() | |||
var unHitEntryPaths []string | |||
@@ -144,7 +144,7 @@ func GetLastCommitForPaths(commit *Commit, treePath string, paths []string) ([]* | |||
} | |||
}() | |||
batchStdinWriter, batchReader, cancel := CatFileBatch(commit.repo.Path) | |||
batchStdinWriter, batchReader, cancel := commit.repo.CatFileBatch() | |||
defer cancel() | |||
mapsize := 4096 | |||
@@ -237,6 +237,10 @@ revListLoop: | |||
// FIXME: is there any order to the way strings are emitted from cat-file? | |||
// if there is - then we could skip once we've passed all of our data | |||
} | |||
if _, err := batchReader.Discard(1); err != nil { | |||
return nil, err | |||
} | |||
break treeReadingLoop | |||
} | |||
@@ -281,6 +285,9 @@ revListLoop: | |||
return nil, err | |||
} | |||
} | |||
if _, err := batchReader.Discard(1); err != nil { | |||
return nil, err | |||
} | |||
// if we haven't found a treeID for the target directory our search is over | |||
if len(treeID) == 0 { | |||
@@ -345,6 +352,9 @@ revListLoop: | |||
if err != nil { | |||
return nil, err | |||
} | |||
if _, err := batchReader.Discard(1); err != nil { | |||
return nil, err | |||
} | |||
commitCommits[i] = c | |||
} | |||
@@ -8,7 +8,6 @@ package git | |||
import ( | |||
"bufio" | |||
"io" | |||
"path" | |||
) | |||
@@ -36,7 +35,7 @@ func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl func() int64, | |||
} | |||
// Get get the last commit information by commit id and entry path | |||
func (c *LastCommitCache) Get(ref, entryPath string, wr *io.PipeWriter, rd *bufio.Reader) (interface{}, error) { | |||
func (c *LastCommitCache) Get(ref, entryPath string, wr WriteCloserError, rd *bufio.Reader) (interface{}, error) { | |||
v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath)) | |||
if vs, ok := v.(string); ok { | |||
log("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs) |
@@ -43,11 +43,18 @@ func GetNote(repo *Repository, commitID string, note *Note) error { | |||
if err != nil { | |||
return err | |||
} | |||
defer dataRc.Close() | |||
closed := false | |||
defer func() { | |||
if !closed { | |||
_ = dataRc.Close() | |||
} | |||
}() | |||
d, err := ioutil.ReadAll(dataRc) | |||
if err != nil { | |||
return err | |||
} | |||
_ = dataRc.Close() | |||
closed = true | |||
note.Message = d | |||
treePath := "" |
@@ -7,8 +7,10 @@ | |||
package git | |||
import ( | |||
"bufio" | |||
"bytes" | |||
"fmt" | |||
"io" | |||
"strconv" | |||
"strings" | |||
) | |||
@@ -86,3 +88,49 @@ func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) { | |||
} | |||
return entries, nil | |||
} | |||
func catBatchParseTreeEntries(ptree *Tree, rd *bufio.Reader, sz int64) ([]*TreeEntry, error) { | |||
fnameBuf := make([]byte, 4096) | |||
modeBuf := make([]byte, 40) | |||
shaBuf := make([]byte, 40) | |||
entries := make([]*TreeEntry, 0, 10) | |||
loop: | |||
for sz > 0 { | |||
mode, fname, sha, count, err := ParseTreeLine(rd, modeBuf, fnameBuf, shaBuf) | |||
if err != nil { | |||
if err == io.EOF { | |||
break loop | |||
} | |||
return nil, err | |||
} | |||
sz -= int64(count) | |||
entry := new(TreeEntry) | |||
entry.ptree = ptree | |||
switch string(mode) { | |||
case "100644": | |||
entry.entryMode = EntryModeBlob | |||
case "100755": | |||
entry.entryMode = EntryModeExec | |||
case "120000": | |||
entry.entryMode = EntryModeSymlink | |||
case "160000": | |||
entry.entryMode = EntryModeCommit | |||
case "40000": | |||
entry.entryMode = EntryModeTree | |||
default: | |||
log("Unknown mode: %v", string(mode)) | |||
return nil, fmt.Errorf("unknown mode: %v", string(mode)) | |||
} | |||
entry.ID = MustID(sha) | |||
entry.name = string(fname) | |||
entries = append(entries, entry) | |||
} | |||
if _, err := rd.Discard(1); err != nil { | |||
return entries, err | |||
} | |||
return entries, nil | |||
} |
@@ -43,8 +43,6 @@ func FindLFSFile(repo *git.Repository, hash git.SHA1) ([]*LFSResult, error) { | |||
basePath := repo.Path | |||
hashStr := hash.String() | |||
// Use rev-list to provide us with all commits in order | |||
revListReader, revListWriter := io.Pipe() | |||
defer func() { | |||
@@ -64,7 +62,7 @@ func FindLFSFile(repo *git.Repository, hash git.SHA1) ([]*LFSResult, error) { | |||
// Next feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary. | |||
// so let's create a batch stdin and stdout | |||
batchStdinWriter, batchReader, cancel := git.CatFileBatch(repo.Path) | |||
batchStdinWriter, batchReader, cancel := repo.CatFileBatch() | |||
defer cancel() | |||
// We'll use a scanner for the revList because it's simpler than a bufio.Reader | |||
@@ -132,8 +130,7 @@ func FindLFSFile(repo *git.Repository, hash git.SHA1) ([]*LFSResult, error) { | |||
return nil, err | |||
} | |||
n += int64(count) | |||
sha := git.To40ByteSHA(sha20byte) | |||
if bytes.Equal(sha, []byte(hashStr)) { | |||
if bytes.Equal(sha20byte, hash[:]) { | |||
result := LFSResult{ | |||
Name: curPath + string(fname), | |||
SHA: curCommit.ID.String(), | |||
@@ -143,7 +140,7 @@ func FindLFSFile(repo *git.Repository, hash git.SHA1) ([]*LFSResult, error) { | |||
} | |||
resultsMap[curCommit.ID.String()+":"+curPath+string(fname)] = &result | |||
} else if string(mode) == git.EntryModeTree.String() { | |||
trees = append(trees, sha) | |||
trees = append(trees, git.To40ByteSHA(sha20byte)) | |||
paths = append(paths, curPath+string(fname)+"/") | |||
} | |||
} |
@@ -8,6 +8,8 @@ | |||
package git | |||
import ( | |||
"bufio" | |||
"context" | |||
"errors" | |||
"path/filepath" | |||
) | |||
@@ -19,6 +21,14 @@ type Repository struct { | |||
tagCache *ObjectCache | |||
gpgSettings *GPGSettings | |||
batchCancel context.CancelFunc | |||
batchReader *bufio.Reader | |||
batchWriter WriteCloserError | |||
checkCancel context.CancelFunc | |||
checkReader *bufio.Reader | |||
checkWriter WriteCloserError | |||
} | |||
// OpenRepository opens the repository at the given path. | |||
@@ -29,12 +39,51 @@ func OpenRepository(repoPath string) (*Repository, error) { | |||
} else if !isDir(repoPath) { | |||
return nil, errors.New("no such file or directory") | |||
} | |||
return &Repository{ | |||
repo := &Repository{ | |||
Path: repoPath, | |||
tagCache: newObjectCache(), | |||
}, nil | |||
} | |||
repo.batchWriter, repo.batchReader, repo.batchCancel = CatFileBatch(repoPath) | |||
repo.checkWriter, repo.checkReader, repo.checkCancel = CatFileBatchCheck(repo.Path) | |||
return repo, nil | |||
} | |||
// CatFileBatch obtains a CatFileBatch for this repository | |||
func (repo *Repository) CatFileBatch() (WriteCloserError, *bufio.Reader, func()) { | |||
if repo.batchCancel == nil || repo.batchReader.Buffered() > 0 { | |||
log("Opening temporary cat file batch for: %s", repo.Path) | |||
return CatFileBatch(repo.Path) | |||
} | |||
return repo.batchWriter, repo.batchReader, func() {} | |||
} | |||
// CatFileBatchCheck obtains a CatFileBatchCheck for this repository | |||
func (repo *Repository) CatFileBatchCheck() (WriteCloserError, *bufio.Reader, func()) { | |||
if repo.checkCancel == nil || repo.checkReader.Buffered() > 0 { | |||
log("Opening temporary cat file batch-check: %s", repo.Path) | |||
return CatFileBatchCheck(repo.Path) | |||
} | |||
return repo.checkWriter, repo.checkReader, func() {} | |||
} | |||
// Close this repository, in particular close the underlying gogitStorage if this is not nil | |||
func (repo *Repository) Close() { | |||
if repo == nil { | |||
return | |||
} | |||
if repo.batchCancel != nil { | |||
repo.batchCancel() | |||
repo.batchReader = nil | |||
repo.batchWriter = nil | |||
repo.batchCancel = nil | |||
} | |||
if repo.checkCancel != nil { | |||
repo.checkCancel() | |||
repo.checkCancel = nil | |||
repo.checkReader = nil | |||
repo.checkWriter = nil | |||
} | |||
} |
@@ -11,7 +11,7 @@ func (repo *Repository) getBlob(id SHA1) (*Blob, error) { | |||
return nil, ErrNotExist{id.String(), ""} | |||
} | |||
return &Blob{ | |||
ID: id, | |||
repoPath: repo.Path, | |||
ID: id, | |||
repo: repo, | |||
}, nil | |||
} |
@@ -33,9 +33,9 @@ func TestRepository_GetBlob_Found(t *testing.T) { | |||
dataReader, err := blob.DataAsync() | |||
assert.NoError(t, err) | |||
defer dataReader.Close() | |||
data, err := ioutil.ReadAll(dataReader) | |||
assert.NoError(t, dataReader.Close()) | |||
assert.NoError(t, err) | |||
assert.Equal(t, testCase.Data, data) | |||
} |
@@ -13,12 +13,30 @@ import ( | |||
"strings" | |||
) | |||
// IsReferenceExist returns true if given reference exists in the repository. | |||
func (repo *Repository) IsReferenceExist(name string) bool { | |||
if name == "" { | |||
return false | |||
} | |||
wr, rd, cancel := repo.CatFileBatchCheck() | |||
defer cancel() | |||
_, err := wr.Write([]byte(name + "\n")) | |||
if err != nil { | |||
log("Error writing to CatFileBatchCheck %v", err) | |||
return false | |||
} | |||
_, _, _, err = ReadBatchLine(rd) | |||
return err == nil | |||
} | |||
// IsBranchExist returns true if given branch exists in current repository. | |||
func (repo *Repository) IsBranchExist(name string) bool { | |||
if name == "" { | |||
return false | |||
} | |||
return IsReferenceExist(repo.Path, BranchPrefix+name) | |||
return repo.IsReferenceExist(BranchPrefix + name) | |||
} | |||
// GetBranches returns branches from the repository, skipping skip initial branches and |
@@ -24,27 +24,6 @@ func (repo *Repository) GetTagCommitID(name string) (string, error) { | |||
return repo.GetRefCommitID(TagPrefix + name) | |||
} | |||
// ConvertToSHA1 returns a Hash object from a potential ID string | |||
func (repo *Repository) ConvertToSHA1(commitID string) (SHA1, error) { | |||
if len(commitID) == 40 { | |||
sha1, err := NewIDFromString(commitID) | |||
if err == nil { | |||
return sha1, nil | |||
} | |||
} | |||
actualCommitID, err := NewCommand("rev-parse", "--verify", commitID).RunInDir(repo.Path) | |||
if err != nil { | |||
if strings.Contains(err.Error(), "unknown revision or path") || | |||
strings.Contains(err.Error(), "fatal: Needed a single revision") { | |||
return SHA1{}, ErrNotExist{commitID, ""} | |||
} | |||
return SHA1{}, err | |||
} | |||
return NewIDFromString(actualCommitID) | |||
} | |||
// GetCommit returns commit object of by ID string. | |||
func (repo *Repository) GetCommit(commitID string) (*Commit, error) { | |||
id, err := repo.ConvertToSHA1(commitID) |
@@ -30,6 +30,27 @@ func (repo *Repository) GetRefCommitID(name string) (string, error) { | |||
return ref.Hash().String(), nil | |||
} | |||
// ConvertToSHA1 returns a Hash object from a potential ID string | |||
func (repo *Repository) ConvertToSHA1(commitID string) (SHA1, error) { | |||
if len(commitID) == 40 { | |||
sha1, err := NewIDFromString(commitID) | |||
if err == nil { | |||
return sha1, nil | |||
} | |||
} | |||
actualCommitID, err := NewCommand("rev-parse", "--verify", commitID).RunInDir(repo.Path) | |||
if err != nil { | |||
if strings.Contains(err.Error(), "unknown revision or path") || | |||
strings.Contains(err.Error(), "fatal: Needed a single revision") { | |||
return SHA1{}, ErrNotExist{commitID, ""} | |||
} | |||
return SHA1{}, err | |||
} | |||
return NewIDFromString(actualCommitID) | |||
} | |||
// IsCommitExist returns true if given commit exists in current repository. | |||
func (repo *Repository) IsCommitExist(name string) bool { | |||
hash := plumbing.NewHash(name) |
@@ -11,8 +11,6 @@ import ( | |||
"errors" | |||
"io" | |||
"io/ioutil" | |||
"os" | |||
"path/filepath" | |||
"strings" | |||
) | |||
@@ -35,27 +33,15 @@ func (repo *Repository) ResolveReference(name string) (string, error) { | |||
// GetRefCommitID returns the last commit ID string of given reference (branch or tag). | |||
func (repo *Repository) GetRefCommitID(name string) (string, error) { | |||
if strings.HasPrefix(name, "refs/") { | |||
// We're gonna try just reading the ref file as this is likely to be quicker than other options | |||
fileInfo, err := os.Lstat(filepath.Join(repo.Path, name)) | |||
if err == nil && fileInfo.Mode().IsRegular() && fileInfo.Size() == 41 { | |||
ref, err := ioutil.ReadFile(filepath.Join(repo.Path, name)) | |||
if err == nil && SHAPattern.Match(ref[:40]) && ref[40] == '\n' { | |||
return string(ref[:40]), nil | |||
} | |||
} | |||
} | |||
stdout, err := NewCommand("show-ref", "--verify", "--hash", name).RunInDir(repo.Path) | |||
if err != nil { | |||
if strings.Contains(err.Error(), "not a valid ref") { | |||
return "", ErrNotExist{name, ""} | |||
} | |||
return "", err | |||
wr, rd, cancel := repo.CatFileBatchCheck() | |||
defer cancel() | |||
_, _ = wr.Write([]byte(name + "\n")) | |||
shaBs, _, _, err := ReadBatchLine(rd) | |||
if IsErrNotExist(err) { | |||
return "", ErrNotExist{name, ""} | |||
} | |||
return strings.TrimSpace(stdout), nil | |||
return string(shaBs), nil | |||
} | |||
// IsCommitExist returns true if given commit exists in current repository. | |||
@@ -65,31 +51,18 @@ func (repo *Repository) IsCommitExist(name string) bool { | |||
} | |||
func (repo *Repository) getCommit(id SHA1) (*Commit, error) { | |||
stdoutReader, stdoutWriter := io.Pipe() | |||
defer func() { | |||
_ = stdoutReader.Close() | |||
_ = stdoutWriter.Close() | |||
}() | |||
go func() { | |||
stderr := strings.Builder{} | |||
err := NewCommand("cat-file", "--batch").RunInDirFullPipeline(repo.Path, stdoutWriter, &stderr, strings.NewReader(id.String()+"\n")) | |||
if err != nil { | |||
_ = stdoutWriter.CloseWithError(ConcatenateError(err, (&stderr).String())) | |||
} else { | |||
_ = stdoutWriter.Close() | |||
} | |||
}() | |||
wr, rd, cancel := repo.CatFileBatch() | |||
defer cancel() | |||
bufReader := bufio.NewReader(stdoutReader) | |||
_, _ = wr.Write([]byte(id.String() + "\n")) | |||
return repo.getCommitFromBatchReader(bufReader, id) | |||
return repo.getCommitFromBatchReader(rd, id) | |||
} | |||
func (repo *Repository) getCommitFromBatchReader(bufReader *bufio.Reader, id SHA1) (*Commit, error) { | |||
_, typ, size, err := ReadBatchLine(bufReader) | |||
func (repo *Repository) getCommitFromBatchReader(rd *bufio.Reader, id SHA1) (*Commit, error) { | |||
_, typ, size, err := ReadBatchLine(rd) | |||
if err != nil { | |||
if errors.Is(err, io.EOF) { | |||
if errors.Is(err, io.EOF) || IsErrNotExist(err) { | |||
return nil, ErrNotExist{ID: id.String()} | |||
} | |||
return nil, err | |||
@@ -101,7 +74,11 @@ func (repo *Repository) getCommitFromBatchReader(bufReader *bufio.Reader, id SHA | |||
case "tag": | |||
// then we need to parse the tag | |||
// and load the commit | |||
data, err := ioutil.ReadAll(io.LimitReader(bufReader, size)) | |||
data, err := ioutil.ReadAll(io.LimitReader(rd, size)) | |||
if err != nil { | |||
return nil, err | |||
} | |||
_, err = rd.Discard(1) | |||
if err != nil { | |||
return nil, err | |||
} | |||
@@ -122,11 +99,50 @@ func (repo *Repository) getCommitFromBatchReader(bufReader *bufio.Reader, id SHA | |||
return commit, nil | |||
case "commit": | |||
return CommitFromReader(repo, id, io.LimitReader(bufReader, size)) | |||
commit, err := CommitFromReader(repo, id, io.LimitReader(rd, size)) | |||
if err != nil { | |||
return nil, err | |||
} | |||
_, err = rd.Discard(1) | |||
if err != nil { | |||
return nil, err | |||
} | |||
return commit, nil | |||
default: | |||
log("Unknown typ: %s", typ) | |||
_, err = rd.Discard(int(size) + 1) | |||
if err != nil { | |||
return nil, err | |||
} | |||
return nil, ErrNotExist{ | |||
ID: id.String(), | |||
} | |||
} | |||
} | |||
// ConvertToSHA1 returns a Hash object from a potential ID string | |||
func (repo *Repository) ConvertToSHA1(commitID string) (SHA1, error) { | |||
if len(commitID) == 40 && SHAPattern.MatchString(commitID) { | |||
sha1, err := NewIDFromString(commitID) | |||
if err == nil { | |||
return sha1, nil | |||
} | |||
} | |||
wr, rd, cancel := repo.CatFileBatchCheck() | |||
defer cancel() | |||
_, err := wr.Write([]byte(commitID + "\n")) | |||
if err != nil { | |||
return SHA1{}, err | |||
} | |||
sha, _, _, err := ReadBatchLine(rd) | |||
if err != nil { | |||
if IsErrNotExist(err) { | |||
return SHA1{}, ErrNotExist{commitID, ""} | |||
} | |||
return SHA1{}, err | |||
} | |||
return MustIDFromString(string(sha)), nil | |||
} |
@@ -21,7 +21,7 @@ import ( | |||
func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) { | |||
// We will feed the commit IDs in order into cat-file --batch, followed by blobs as necessary. | |||
// so let's create a batch stdin and stdout | |||
batchStdinWriter, batchReader, cancel := CatFileBatch(repo.Path) | |||
batchStdinWriter, batchReader, cancel := repo.CatFileBatch() | |||
defer cancel() | |||
writeID := func(id string) error { |
@@ -9,7 +9,11 @@ package git | |||
// IsTagExist returns true if given tag exists in the repository. | |||
func (repo *Repository) IsTagExist(name string) bool { | |||
return IsReferenceExist(repo.Path, TagPrefix+name) | |||
if name == "" { | |||
return false | |||
} | |||
return repo.IsReferenceExist(TagPrefix + name) | |||
} | |||
// GetTags returns all tags of the repository. |
@@ -7,33 +7,18 @@ | |||
package git | |||
import ( | |||
"bufio" | |||
"fmt" | |||
"io" | |||
"io/ioutil" | |||
"strings" | |||
) | |||
func (repo *Repository) getTree(id SHA1) (*Tree, error) { | |||
stdoutReader, stdoutWriter := io.Pipe() | |||
defer func() { | |||
_ = stdoutReader.Close() | |||
_ = stdoutWriter.Close() | |||
}() | |||
wr, rd, cancel := repo.CatFileBatch() | |||
defer cancel() | |||
go func() { | |||
stderr := &strings.Builder{} | |||
err := NewCommand("cat-file", "--batch").RunInDirFullPipeline(repo.Path, stdoutWriter, stderr, strings.NewReader(id.String()+"\n")) | |||
if err != nil { | |||
_ = stdoutWriter.CloseWithError(ConcatenateError(err, stderr.String())) | |||
} else { | |||
_ = stdoutWriter.Close() | |||
} | |||
}() | |||
_, _ = wr.Write([]byte(id.String() + "\n")) | |||
bufReader := bufio.NewReader(stdoutReader) | |||
// ignore the SHA | |||
_, typ, size, err := ReadBatchLine(bufReader) | |||
_, typ, size, err := ReadBatchLine(rd) | |||
if err != nil { | |||
return nil, err | |||
} | |||
@@ -41,7 +26,7 @@ func (repo *Repository) getTree(id SHA1) (*Tree, error) { | |||
switch typ { | |||
case "tag": | |||
resolvedID := id | |||
data, err := ioutil.ReadAll(io.LimitReader(bufReader, size)) | |||
data, err := ioutil.ReadAll(io.LimitReader(rd, size)) | |||
if err != nil { | |||
return nil, err | |||
} | |||
@@ -54,24 +39,27 @@ func (repo *Repository) getTree(id SHA1) (*Tree, error) { | |||
return nil, err | |||
} | |||
commit.Tree.ResolvedID = resolvedID | |||
log("tag.commit.Tree: %s %v", commit.Tree.ID.String(), commit.Tree.repo) | |||
return &commit.Tree, nil | |||
case "commit": | |||
commit, err := CommitFromReader(repo, id, io.LimitReader(bufReader, size)) | |||
commit, err := CommitFromReader(repo, id, io.LimitReader(rd, size)) | |||
if err != nil { | |||
_ = stdoutReader.CloseWithError(err) | |||
return nil, err | |||
} | |||
if _, err := rd.Discard(1); err != nil { | |||
return nil, err | |||
} | |||
commit.Tree.ResolvedID = commit.ID | |||
log("commit.Tree: %s %v", commit.Tree.ID.String(), commit.Tree.repo) | |||
return &commit.Tree, nil | |||
case "tree": | |||
stdoutReader.Close() | |||
tree := NewTree(repo, id) | |||
tree.ResolvedID = id | |||
tree.entries, err = catBatchParseTreeEntries(tree, rd, size) | |||
if err != nil { | |||
return nil, err | |||
} | |||
tree.entriesParsed = true | |||
return tree, nil | |||
default: | |||
_ = stdoutReader.CloseWithError(fmt.Errorf("unknown typ: %s", typ)) | |||
return nil, ErrNotExist{ | |||
ID: id.String(), | |||
} | |||
@@ -81,12 +69,12 @@ func (repo *Repository) getTree(id SHA1) (*Tree, error) { | |||
// GetTree find the tree object in the repository. | |||
func (repo *Repository) GetTree(idStr string) (*Tree, error) { | |||
if len(idStr) != 40 { | |||
res, err := NewCommand("rev-parse", "--verify", idStr).RunInDir(repo.Path) | |||
res, err := repo.GetRefCommitID(idStr) | |||
if err != nil { | |||
return nil, err | |||
} | |||
if len(res) > 0 { | |||
idStr = res[:len(res)-1] | |||
idStr = res | |||
} | |||
} | |||
id, err := NewIDFromString(idStr) |
@@ -15,6 +15,7 @@ import ( | |||
func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) { | |||
if len(relpath) == 0 { | |||
return &TreeEntry{ | |||
ptree: t, | |||
ID: t.ID, | |||
name: "", | |||
fullName: "", |
@@ -34,12 +34,19 @@ func (te *TreeEntry) FollowLink() (*TreeEntry, error) { | |||
if err != nil { | |||
return nil, err | |||
} | |||
defer r.Close() | |||
closed := false | |||
defer func() { | |||
if !closed { | |||
_ = r.Close() | |||
} | |||
}() | |||
buf := make([]byte, te.Size()) | |||
_, err = io.ReadFull(r, buf) | |||
if err != nil { | |||
return nil, err | |||
} | |||
_ = r.Close() | |||
closed = true | |||
lnk := string(buf) | |||
t := te.ptree |
@@ -84,10 +84,10 @@ func (te *TreeEntry) IsExecutable() bool { | |||
// Blob returns the blob object the entry | |||
func (te *TreeEntry) Blob() *Blob { | |||
return &Blob{ | |||
ID: te.ID, | |||
repoPath: te.ptree.repo.Path, | |||
name: te.Name(), | |||
size: te.size, | |||
gotSize: te.sized, | |||
ID: te.ID, | |||
name: te.Name(), | |||
size: te.size, | |||
gotSize: te.sized, | |||
repo: te.ptree.repo, | |||
} | |||
} |
@@ -7,6 +7,8 @@ | |||
package git | |||
import ( | |||
"io" | |||
"math" | |||
"strings" | |||
) | |||
@@ -32,6 +34,52 @@ func (t *Tree) ListEntries() (Entries, error) { | |||
return t.entries, nil | |||
} | |||
if t.repo != nil { | |||
wr, rd, cancel := t.repo.CatFileBatch() | |||
defer cancel() | |||
_, _ = wr.Write([]byte(t.ID.String() + "\n")) | |||
_, typ, sz, err := ReadBatchLine(rd) | |||
if err != nil { | |||
return nil, err | |||
} | |||
if typ == "commit" { | |||
treeID, err := ReadTreeID(rd, sz) | |||
if err != nil && err != io.EOF { | |||
return nil, err | |||
} | |||
_, _ = wr.Write([]byte(treeID + "\n")) | |||
_, typ, sz, err = ReadBatchLine(rd) | |||
if err != nil { | |||
return nil, err | |||
} | |||
} | |||
if typ == "tree" { | |||
t.entries, err = catBatchParseTreeEntries(t, rd, sz) | |||
if err != nil { | |||
return nil, err | |||
} | |||
t.entriesParsed = true | |||
return t.entries, nil | |||
} | |||
// Not a tree just use ls-tree instead | |||
for sz > math.MaxInt32 { | |||
discarded, err := rd.Discard(math.MaxInt32) | |||
sz -= int64(discarded) | |||
if err != nil { | |||
return nil, err | |||
} | |||
} | |||
for sz > 0 { | |||
discarded, err := rd.Discard(int(sz)) | |||
sz -= int64(discarded) | |||
if err != nil { | |||
return nil, err | |||
} | |||
} | |||
} | |||
stdout, err := NewCommand("ls-tree", "-l", t.ID.String()).RunInDirBytes(t.repo.Path) | |||
if err != nil { | |||
if strings.Contains(err.Error(), "fatal: Not a valid object name") || strings.Contains(err.Error(), "fatal: not a tree object") { |
@@ -176,7 +176,7 @@ func NewBleveIndexer(indexDir string) (*BleveIndexer, bool, error) { | |||
return indexer, created, err | |||
} | |||
func (b *BleveIndexer) addUpdate(batchWriter *io.PipeWriter, batchReader *bufio.Reader, commitSha string, update fileUpdate, repo *models.Repository, batch rupture.FlushingBatch) error { | |||
func (b *BleveIndexer) addUpdate(batchWriter git.WriteCloserError, batchReader *bufio.Reader, commitSha string, update fileUpdate, repo *models.Repository, batch rupture.FlushingBatch) error { | |||
// Ignore vendored files in code search | |||
if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) { | |||
return nil |
@@ -175,7 +175,7 @@ func (b *ElasticSearchIndexer) init() (bool, error) { | |||
return exists, nil | |||
} | |||
func (b *ElasticSearchIndexer) addUpdate(batchWriter *io.PipeWriter, batchReader *bufio.Reader, sha string, update fileUpdate, repo *models.Repository) ([]elastic.BulkableRequest, error) { | |||
func (b *ElasticSearchIndexer) addUpdate(batchWriter git.WriteCloserError, batchReader *bufio.Reader, sha string, update fileUpdate, repo *models.Repository) ([]elastic.BulkableRequest, error) { | |||
// Ignore vendored files in code search | |||
if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) { | |||
return nil, nil |
@@ -100,7 +100,11 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob) error { | |||
if err != nil { | |||
return err | |||
} | |||
closed := false | |||
defer func() { | |||
if closed { | |||
return | |||
} | |||
if err = dataRc.Close(); err != nil { | |||
log.Error("ServeBlobOrLFS: Close: %v", err) | |||
} | |||
@@ -110,6 +114,10 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob) error { | |||
if pointer.IsValid() { | |||
meta, _ := ctx.Repo.Repository.GetLFSMetaObjectByOid(pointer.Oid) | |||
if meta == nil { | |||
if err = dataRc.Close(); err != nil { | |||
log.Error("ServeBlobOrLFS: Close: %v", err) | |||
} | |||
closed = true | |||
return ServeBlob(ctx, blob) | |||
} | |||
if httpcache.HandleGenericETagCache(ctx.Req, ctx.Resp, `"`+pointer.Oid+`"`) { | |||
@@ -126,6 +134,10 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob) error { | |||
}() | |||
return ServeData(ctx, ctx.Repo.TreePath, meta.Size, lfsDataRc) | |||
} | |||
if err = dataRc.Close(); err != nil { | |||
log.Error("ServeBlobOrLFS: Close: %v", err) | |||
} | |||
closed = true | |||
return ServeBlob(ctx, blob) | |||
} |