Improve listing performance by using go-git (#6478)

* Use go-git for tree reading and commit info lookup.

Signed-off-by: Filip Navara <navara@emclient.com>

* Use TreeEntry.IsRegular() instead of ObjectType that was removed.

Signed-off-by: Filip Navara <navara@emclient.com>

* Use the treePath to optimize commit info search.

Signed-off-by: Filip Navara <navara@emclient.com>

* Extract the latest commit at treePath along with the other commits.

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix listing commit info for a directory that was created in one commit and never modified after.

Signed-off-by: Filip Navara <navara@emclient.com>

* Avoid nearly all external 'git' invocations when doing directory listing (.editorconfig code path is still hit).

Signed-off-by: Filip Navara <navara@emclient.com>

* Use go-git for reading blobs.

Signed-off-by: Filip Navara <navara@emclient.com>

* Make SHA1 type alias for plumbing.Hash in go-git.

Signed-off-by: Filip Navara <navara@emclient.com>

* Make Signature type alias for object.Signature in go-git.

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix GetCommitsInfo for repository with only one commit.

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix PGP signature verification.

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix issues with walking commit graph across merges.

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix typo in condition.

Signed-off-by: Filip Navara <navara@emclient.com>

* Speed up loading branch list by keeping the repository reference (and thus all the loaded packfile indexes).

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix lising submodules.

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix build

Signed-off-by: Filip Navara <navara@emclient.com>

* Add back commit cache because of name-rev

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix tests

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix code style

* Fix spelling

* Address PR feedback

Signed-off-by: Filip Navara <navara@emclient.com>

* Update vendor module list

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix getting trees by commit id

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix remaining unit test failures

* Fix GetTreeBySHA

* Avoid running `git name-rev` if not necessary

Signed-off-by: Filip Navara <navara@emclient.com>

* Move Branch code to git module

* Clean up GPG signature verification and fix it for tagged commits

* Address PR feedback (import formatting, copyright headers)

* Make blob lookup by SHA working

* Update tests to use public API

* Allow getting content from any type of object through the blob interface

* Change test to actually expect the object content that is in the GIT repository

* Change one more test to actually expect the object content that is in the GIT repository

* Add comments

7 files changed, 20 insertions, 24 deletions

diff --git a/routers/repo/commit.go b/routers/repo/commit.go
index a1e9b8bc5d..9263bcac7f 100644
--- a/routers/repo/commit.go
+++ b/routers/repo/commit.go
@@ -243,6 +243,7 @@ func Diff(ctx *context.Context) {
 		ctx.Data["BeforeSourcePath"] = setting.AppSubURL + "/" + path.Join(userName, repoName, "src", "commit", parents[0])
 	}
 	ctx.Data["RawPath"] = setting.AppSubURL + "/" + path.Join(userName, repoName, "raw", "commit", commitID)
+	ctx.Data["BranchName"], err = commit.GetBranchName()
 	ctx.HTML(200, tplDiff)
 }
 
diff --git a/routers/repo/editor.go b/routers/repo/editor.go
index b5bb2f0ce3..4314218bf3 100644
--- a/routers/repo/editor.go
+++ b/routers/repo/editor.go
@@ -95,11 +95,12 @@ func editFile(ctx *context.Context, isNewFile bool) {
 			return
 		}
 
-		dataRc, err := blob.Data()
+		dataRc, err := blob.DataAsync()
 		if err != nil {
 			ctx.NotFound("blob.Data", err)
 			return
 		}
+		defer dataRc.Close()
 
 		ctx.Data["FileSize"] = blob.Size()
 		ctx.Data["FileName"] = blob.Name()
@@ -251,9 +252,9 @@ func editFilePost(ctx *context.Context, form auth.EditRepoFileForm, isNewFile bo
 		} else if models.IsErrRepoFileAlreadyExists(err) {
 			ctx.Data["Err_TreePath"] = true
 			ctx.RenderWithErr(ctx.Tr("repo.editor.file_already_exists", form.TreePath), tplEditFile, &form)
-		} else if models.IsErrBranchNotExist(err) {
+		} else if git.IsErrBranchNotExist(err) {
 			// For when a user adds/updates a file to a branch that no longer exists
-			if branchErr, ok := err.(models.ErrBranchNotExist); ok {
+			if branchErr, ok := err.(git.ErrBranchNotExist); ok {
 				ctx.RenderWithErr(ctx.Tr("repo.editor.branch_does_not_exist", branchErr.Name), tplEditFile, &form)
 			} else {
 				ctx.Error(500, err.Error())
@@ -417,9 +418,9 @@ func DeleteFilePost(ctx *context.Context, form auth.DeleteRepoFileForm) {
 			} else {
 				ctx.ServerError("DeleteRepoFile", err)
 			}
-		} else if models.IsErrBranchNotExist(err) {
+		} else if git.IsErrBranchNotExist(err) {
 			// For when a user deletes a file to a branch that no longer exists
-			if branchErr, ok := err.(models.ErrBranchNotExist); ok {
+			if branchErr, ok := err.(git.ErrBranchNotExist); ok {
 				ctx.RenderWithErr(ctx.Tr("repo.editor.branch_does_not_exist", branchErr.Name), tplEditFile, &form)
 			} else {
 				ctx.Error(500, err.Error())
diff --git a/routers/repo/issue.go b/routers/repo/issue.go
index 42f3ddf4e8..7d235d84ef 100644
--- a/routers/repo/issue.go
+++ b/routers/repo/issue.go
@@ -9,7 +9,6 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"net/http"
 	"strconv"
@@ -363,7 +362,6 @@ func RetrieveRepoMetas(ctx *context.Context, repo *models.Repository) []*models.
 }
 
 func getFileContentFromDefaultBranch(ctx *context.Context, filename string) (string, bool) {
-	var r io.Reader
 	var bytes []byte
 
 	if ctx.Repo.Commit == nil {
@@ -381,10 +379,11 @@ func getFileContentFromDefaultBranch(ctx *context.Context, filename string) (str
 	if entry.Blob().Size() >= setting.UI.MaxDisplayFileSize {
 		return "", false
 	}
-	r, err = entry.Blob().Data()
+	r, err := entry.Blob().DataAsync()
 	if err != nil {
 		return "", false
 	}
+	defer r.Close()
 	bytes, err = ioutil.ReadAll(r)
 	if err != nil {
 		return "", false
diff --git a/routers/repo/setting_protected_branch.go b/routers/repo/setting_protected_branch.go
index 05ad0ec0ec..b5a115b6a4 100644
--- a/routers/repo/setting_protected_branch.go
+++ b/routers/repo/setting_protected_branch.go
@@ -103,7 +103,7 @@ func SettingsProtectedBranch(c *context.Context) {
 
 	protectBranch, err := models.GetProtectedBranchBy(c.Repo.Repository.ID, branch)
 	if err != nil {
-		if !models.IsErrBranchNotExist(err) {
+		if !git.IsErrBranchNotExist(err) {
 			c.ServerError("GetProtectBranchOfRepoByName", err)
 			return
 		}
@@ -152,7 +152,7 @@ func SettingsProtectedBranchPost(ctx *context.Context, f auth.ProtectBranchForm)
 
 	protectBranch, err := models.GetProtectedBranchBy(ctx.Repo.Repository.ID, branch)
 	if err != nil {
-		if !models.IsErrBranchNotExist(err) {
+		if !git.IsErrBranchNotExist(err) {
 			ctx.ServerError("GetProtectBranchOfRepoByName", err)
 			return
 		}
diff --git a/routers/repo/view.go b/routers/repo/view.go
index e883ebbf89..d20f94dfba 100644
--- a/routers/repo/view.go
+++ b/routers/repo/view.go
@@ -49,7 +49,8 @@ func renderDirectory(ctx *context.Context, treeLink string) {
 	}
 	entries.CustomSort(base.NaturalSortLess)
 
-	ctx.Data["Files"], err = entries.GetCommitsInfo(ctx.Repo.Commit, ctx.Repo.TreePath, nil)
+	var latestCommit *git.Commit
+	ctx.Data["Files"], latestCommit, err = entries.GetCommitsInfo(ctx.Repo.Commit, ctx.Repo.TreePath, nil)
 	if err != nil {
 		ctx.ServerError("GetCommitsInfo", err)
 		return
@@ -178,14 +179,6 @@ func renderDirectory(ctx *context.Context, treeLink string) {
 
 	// Show latest commit info of repository in table header,
 	// or of directory if not in root directory.
-	latestCommit := ctx.Repo.Commit
-	if len(ctx.Repo.TreePath) > 0 {
-		latestCommit, err = ctx.Repo.Commit.GetCommitByPath(ctx.Repo.TreePath)
-		if err != nil {
-			ctx.ServerError("GetCommitByPath", err)
-			return
-		}
-	}
 	ctx.Data["LatestCommit"] = latestCommit
 	ctx.Data["LatestCommitVerification"] = models.ParseCommitWithSignature(latestCommit)
 	ctx.Data["LatestCommitUser"] = models.ValidateCommitWithEmail(latestCommit)
diff --git a/routers/repo/wiki.go b/routers/repo/wiki.go
index 790214e8e5..c0fb370dd9 100644
--- a/routers/repo/wiki.go
+++ b/routers/repo/wiki.go
@@ -57,7 +57,7 @@ func findEntryForFile(commit *git.Commit, target string) (*git.TreeEntry, error)
 		return nil, err
 	}
 	for _, entry := range entries {
-		if entry.Type == git.ObjectBlob && entry.Name() == target {
+		if entry.IsRegular() && entry.Name() == target {
 			return entry, nil
 		}
 	}
@@ -81,11 +81,12 @@ func findWikiRepoCommit(ctx *context.Context) (*git.Repository, *git.Commit, err
 // wikiContentsByEntry returns the contents of the wiki page referenced by the
 // given tree entry. Writes to ctx if an error occurs.
 func wikiContentsByEntry(ctx *context.Context, entry *git.TreeEntry) []byte {
-	reader, err := entry.Blob().Data()
+	reader, err := entry.Blob().DataAsync()
 	if err != nil {
 		ctx.ServerError("Blob.Data", err)
 		return nil
 	}
+	defer reader.Close()
 	content, err := ioutil.ReadAll(reader)
 	if err != nil {
 		ctx.ServerError("ReadAll", err)
@@ -125,7 +126,7 @@ func renderWikiPage(ctx *context.Context, isViewPage bool) (*git.Repository, *gi
 		}
 		pages := make([]PageMeta, 0, len(entries))
 		for _, entry := range entries {
-			if entry.Type != git.ObjectBlob {
+			if !entry.IsRegular() {
 				continue
 			}
 			wikiName, err := models.WikiFilenameToName(entry.Name())
@@ -259,7 +260,7 @@ func WikiPages(ctx *context.Context) {
 	}
 	pages := make([]PageMeta, 0, len(entries))
 	for _, entry := range entries {
-		if entry.Type != git.ObjectBlob {
+		if !entry.IsRegular() {
 			continue
 		}
 		c, err := wikiRepo.GetCommitByPath(entry.Name())
diff --git a/routers/repo/wiki_test.go b/routers/repo/wiki_test.go
index 7302f9e48b..4687d24f6b 100644
--- a/routers/repo/wiki_test.go
+++ b/routers/repo/wiki_test.go
@@ -40,8 +40,9 @@ func wikiContent(t *testing.T, repo *models.Repository, wikiName string) string
 	if !assert.NotNil(t, entry) {
 		return ""
 	}
-	reader, err := entry.Blob().Data()
+	reader, err := entry.Blob().DataAsync()
 	assert.NoError(t, err)
+	defer reader.Close()
 	bytes, err := ioutil.ReadAll(reader)
 	assert.NoError(t, err)
 	return string(bytes)