More efficiently parse shas for shaPostProcessor (#16101)

* More efficiently parse shas for shaPostProcessor

The shaPostProcessor currently repeatedly calls git rev-parse --verify on both backends
which is fine if there is only one thing that matches a sha - however if there are
multiple things then this becomes wildly inefficient.

This PR provides functions for both backends which are much faster to use.

Fix #16092

* Add ShaExistCache to RenderContext

Signed-off-by: Andrew Thornton <art27@cantab.net>
Co-authored-by: 6543 <6543@obermui.de>

diff --git a/modules/git/repo_branch_gogit.go b/modules/git/repo_branch_gogit.go
index b00253f6ffd668ec045a952b12cb0daa480e13c5..e8386b2dbd98fade940e4bfd69e143f6539448bd 100644 (file)
--- a/modules/git/repo_branch_gogit.go
+++ b/modules/git/repo_branch_gogit.go
@@ -13,6 +13,30 @@ import (
        "github.com/go-git/go-git/v5/plumbing"
 )
 
+// IsObjectExist returns true if given reference exists in the repository.
+func (repo *Repository) IsObjectExist(name string) bool {
+       if name == "" {
+               return false
+       }
+
+       _, err := repo.gogitRepo.ResolveRevision(plumbing.Revision(name))
+
+       return err == nil
+}
+
+// IsReferenceExist returns true if given reference exists in the repository.
+func (repo *Repository) IsReferenceExist(name string) bool {
+       if name == "" {
+               return false
+       }
+
+       reference, err := repo.gogitRepo.Reference(plumbing.ReferenceName(name), true)
+       if err != nil {
+               return false
+       }
+       return reference.Type() != plumbing.InvalidReference
+}
+
 // IsBranchExist returns true if given branch exists in current repository.
 func (repo *Repository) IsBranchExist(name string) bool {
        if name == "" {

diff --git a/modules/git/repo_branch_nogogit.go b/modules/git/repo_branch_nogogit.go
index 13ddcf06cf65e2f3333877cfd73d55cb47be36ad..dd34e48899034a394f3d7adf97a6b2c39bf7b7e7 100644 (file)
--- a/modules/git/repo_branch_nogogit.go
+++ b/modules/git/repo_branch_nogogit.go
@@ -9,10 +9,28 @@ package git
 
 import (
        "bufio"
+       "bytes"
        "io"
        "strings"
 )
 
+// IsObjectExist returns true if given reference exists in the repository.
+func (repo *Repository) IsObjectExist(name string) bool {
+       if name == "" {
+               return false
+       }
+
+       wr, rd, cancel := repo.CatFileBatchCheck()
+       defer cancel()
+       _, err := wr.Write([]byte(name + "\n"))
+       if err != nil {
+               log("Error writing to CatFileBatchCheck %v", err)
+               return false
+       }
+       sha, _, _, err := ReadBatchLine(rd)
+       return err == nil && bytes.HasPrefix(sha, []byte(strings.TrimSpace(name)))
+}
+
 // IsReferenceExist returns true if given reference exists in the repository.
 func (repo *Repository) IsReferenceExist(name string) bool {
        if name == "" {

diff --git a/modules/markup/html.go b/modules/markup/html.go
index 2a83b8716e0fa1729f029a929df215db872ed3e6..edf860da4510d28b522cce2488186cd2dc528126 100644 (file)
--- a/modules/markup/html.go
+++ b/modules/markup/html.go
@@ -286,6 +286,7 @@ var tagCleaner = regexp.MustCompile(`<((?:/?\w+/\w+)|(?:/[\w ]+/)|(/?[hH][tT][mM
 var nulCleaner = strings.NewReplacer("\000", "")
 
 func postProcess(ctx *RenderContext, procs []processor, input io.Reader, output io.Writer) error {
+       defer ctx.Cancel()
        // FIXME: don't read all content to memory
        rawHTML, err := ioutil.ReadAll(input)
        if err != nil {
@@ -996,6 +997,9 @@ func sha1CurrentPatternProcessor(ctx *RenderContext, node *html.Node) {
 
        start := 0
        next := node.NextSibling
+       if ctx.ShaExistCache == nil {
+               ctx.ShaExistCache = make(map[string]bool)
+       }
        for node != nil && node != next && start < len(node.Data) {
                m := sha1CurrentPattern.FindStringSubmatchIndex(node.Data[start:])
                if m == nil {
@@ -1013,10 +1017,28 @@ func sha1CurrentPatternProcessor(ctx *RenderContext, node *html.Node) {
                // as used by git and github for linking and thus we have to do similar.
                // Because of this, we check to make sure that a matched hash is actually
                // a commit in the repository before making it a link.
-               if _, err := git.NewCommand("rev-parse", "--verify", hash).RunInDirBytes(ctx.Metas["repoPath"]); err != nil {
-                       if !strings.Contains(err.Error(), "fatal: Needed a single revision") {
-                               log.Debug("sha1CurrentPatternProcessor git rev-parse: %v", err)
+
+               // check cache first
+               exist, inCache := ctx.ShaExistCache[hash]
+               if !inCache {
+                       if ctx.GitRepo == nil {
+                               var err error
+                               ctx.GitRepo, err = git.OpenRepository(ctx.Metas["repoPath"])
+                               if err != nil {
+                                       log.Error("unable to open repository: %s Error: %v", ctx.Metas["repoPath"], err)
+                                       return
+                               }
+                               ctx.AddCancel(func() {
+                                       ctx.GitRepo.Close()
+                                       ctx.GitRepo = nil
+                               })
                        }
+
+                       exist = ctx.GitRepo.IsObjectExist(hash)
+                       ctx.ShaExistCache[hash] = exist
+               }
+
+               if !exist {
                        start = m[3]
                        continue
                }

diff --git a/modules/markup/renderer.go b/modules/markup/renderer.go
index 5d35bd5a6771569601ec51409eea0075044de87e..d60c8ad71066b2894cec2f762d0f00190714fff5 100644 (file)
--- a/modules/markup/renderer.go
+++ b/modules/markup/renderer.go
@@ -13,6 +13,7 @@ import (
        "strings"
        "sync"
 
+       "code.gitea.io/gitea/modules/git"
        "code.gitea.io/gitea/modules/setting"
 )
 
@@ -35,13 +36,44 @@ func Init() {
 
 // RenderContext represents a render context
 type RenderContext struct {
-       Ctx         context.Context
-       Filename    string
-       Type        string
-       IsWiki      bool
-       URLPrefix   string
-       Metas       map[string]string
-       DefaultLink string
+       Ctx           context.Context
+       Filename      string
+       Type          string
+       IsWiki        bool
+       URLPrefix     string
+       Metas         map[string]string
+       DefaultLink   string
+       GitRepo       *git.Repository
+       ShaExistCache map[string]bool
+       cancelFn      func()
+}
+
+// Cancel runs any cleanup functions that have been registered for this Ctx
+func (ctx *RenderContext) Cancel() {
+       if ctx == nil {
+               return
+       }
+       ctx.ShaExistCache = map[string]bool{}
+       if ctx.cancelFn == nil {
+               return
+       }
+       ctx.cancelFn()
+}
+
+// AddCancel adds the provided fn as a Cleanup for this Ctx
+func (ctx *RenderContext) AddCancel(fn func()) {
+       if ctx == nil {
+               return
+       }
+       oldCancelFn := ctx.cancelFn
+       if oldCancelFn == nil {
+               ctx.cancelFn = fn
+               return
+       }
+       ctx.cancelFn = func() {
+               defer oldCancelFn()
+               fn()
+       }
 }
 
 // Renderer defines an interface for rendering markup file to HTML

diff --git a/routers/web/org/home.go b/routers/web/org/home.go
index d84ae870ab6dbc83aced60d8ce217f65012b0ee8..ad14f184544474d220b09c72531b27d5ffcbb508 100644 (file)
--- a/routers/web/org/home.go
+++ b/routers/web/org/home.go
@@ -41,6 +41,7 @@ func Home(ctx *context.Context) {
                desc, err := markdown.RenderString(&markup.RenderContext{
                        URLPrefix: ctx.Repo.RepoLink,
                        Metas:     map[string]string{"mode": "document"},
+                       GitRepo:   ctx.Repo.GitRepo,
                }, org.Description)
                if err != nil {
                        ctx.ServerError("RenderString", err)

diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go
index fd2877e7069d6b28f6d8ec10041dd3d1f04d9c46..a7951b6bce1803a91694875562a10e47995946af 100644 (file)
--- a/routers/web/repo/issue.go
+++ b/routers/web/repo/issue.go
@@ -1137,6 +1137,7 @@ func ViewIssue(ctx *context.Context) {
        issue.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
                URLPrefix: ctx.Repo.RepoLink,
                Metas:     ctx.Repo.Repository.ComposeMetas(),
+               GitRepo:   ctx.Repo.GitRepo,
        }, issue.Content)
        if err != nil {
                ctx.ServerError("RenderString", err)
@@ -1301,6 +1302,7 @@ func ViewIssue(ctx *context.Context) {
                        comment.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
                                URLPrefix: ctx.Repo.RepoLink,
                                Metas:     ctx.Repo.Repository.ComposeMetas(),
+                               GitRepo:   ctx.Repo.GitRepo,
                        }, comment.Content)
                        if err != nil {
                                ctx.ServerError("RenderString", err)
@@ -1376,6 +1378,7 @@ func ViewIssue(ctx *context.Context) {
                        comment.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
                                URLPrefix: ctx.Repo.RepoLink,
                                Metas:     ctx.Repo.Repository.ComposeMetas(),
+                               GitRepo:   ctx.Repo.GitRepo,
                        }, comment.Content)
                        if err != nil {
                                ctx.ServerError("RenderString", err)
@@ -1734,6 +1737,7 @@ func UpdateIssueContent(ctx *context.Context) {
        content, err := markdown.RenderString(&markup.RenderContext{
                URLPrefix: ctx.Query("context"),
                Metas:     ctx.Repo.Repository.ComposeMetas(),
+               GitRepo:   ctx.Repo.GitRepo,
        }, issue.Content)
        if err != nil {
                ctx.ServerError("RenderString", err)
@@ -2161,6 +2165,7 @@ func UpdateCommentContent(ctx *context.Context) {
        content, err := markdown.RenderString(&markup.RenderContext{
                URLPrefix: ctx.Query("context"),
                Metas:     ctx.Repo.Repository.ComposeMetas(),
+               GitRepo:   ctx.Repo.GitRepo,
        }, comment.Content)
        if err != nil {
                ctx.ServerError("RenderString", err)

diff --git a/routers/web/repo/milestone.go b/routers/web/repo/milestone.go
index bb6b310cbe8d4784aae2278d4eb6765b14e09fc9..4cdca38dd02b64e18b829963c893df8a2dccbb23 100644 (file)
--- a/routers/web/repo/milestone.go
+++ b/routers/web/repo/milestone.go
@@ -88,6 +88,7 @@ func Milestones(ctx *context.Context) {
                m.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
                        URLPrefix: ctx.Repo.RepoLink,
                        Metas:     ctx.Repo.Repository.ComposeMetas(),
+                       GitRepo:   ctx.Repo.GitRepo,
                }, m.Content)
                if err != nil {
                        ctx.ServerError("RenderString", err)
@@ -280,6 +281,7 @@ func MilestoneIssuesAndPulls(ctx *context.Context) {
        milestone.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
                URLPrefix: ctx.Repo.RepoLink,
                Metas:     ctx.Repo.Repository.ComposeMetas(),
+               GitRepo:   ctx.Repo.GitRepo,
        }, milestone.Content)
        if err != nil {
                ctx.ServerError("RenderString", err)

diff --git a/routers/web/repo/projects.go b/routers/web/repo/projects.go
index eb0719995cb5515de939dd258ef43d1f5428d03e..c7490893d5fe69916f5e3a3b614fad50f7d63c9c 100644 (file)
--- a/routers/web/repo/projects.go
+++ b/routers/web/repo/projects.go
@@ -81,6 +81,7 @@ func Projects(ctx *context.Context) {
                projects[i].RenderedContent, err = markdown.RenderString(&markup.RenderContext{
                        URLPrefix: ctx.Repo.RepoLink,
                        Metas:     ctx.Repo.Repository.ComposeMetas(),
+                       GitRepo:   ctx.Repo.GitRepo,
                }, projects[i].Description)
                if err != nil {
                        ctx.ServerError("RenderString", err)
@@ -322,6 +323,7 @@ func ViewProject(ctx *context.Context) {
        project.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
                URLPrefix: ctx.Repo.RepoLink,
                Metas:     ctx.Repo.Repository.ComposeMetas(),
+               GitRepo:   ctx.Repo.GitRepo,
        }, project.Description)
        if err != nil {
                ctx.ServerError("RenderString", err)

diff --git a/routers/web/repo/release.go b/routers/web/repo/release.go
index b7730e4ee25e938dc738ceae835358ffda200000..3b700e80160c599f5fd39e289dca83bd10d3c6d0 100644 (file)
--- a/routers/web/repo/release.go
+++ b/routers/web/repo/release.go
@@ -145,6 +145,7 @@ func releasesOrTags(ctx *context.Context, isTagList bool) {
                r.Note, err = markdown.RenderString(&markup.RenderContext{
                        URLPrefix: ctx.Repo.RepoLink,
                        Metas:     ctx.Repo.Repository.ComposeMetas(),
+                       GitRepo:   ctx.Repo.GitRepo,
                }, r.Note)
                if err != nil {
                        ctx.ServerError("RenderString", err)
@@ -213,6 +214,7 @@ func SingleRelease(ctx *context.Context) {
        release.Note, err = markdown.RenderString(&markup.RenderContext{
                URLPrefix: ctx.Repo.RepoLink,
                Metas:     ctx.Repo.Repository.ComposeMetas(),
+               GitRepo:   ctx.Repo.GitRepo,
        }, release.Note)
        if err != nil {
                ctx.ServerError("RenderString", err)

diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go
index cd5b0f43edbc7d7e9de836e9a5a46fcc15da7fa7..74e2a2959772471f1919cc09c2455369ecc68147 100644 (file)
--- a/routers/web/repo/view.go
+++ b/routers/web/repo/view.go
@@ -338,6 +338,7 @@ func renderDirectory(ctx *context.Context, treeLink string) {
                                                Filename:  readmeFile.name,
                                                URLPrefix: readmeTreelink,
                                                Metas:     ctx.Repo.Repository.ComposeDocumentMetas(),
+                                               GitRepo:   ctx.Repo.GitRepo,
                                        }, rd, &result)
                                        if err != nil {
                                                log.Error("Render failed: %v then fallback", err)
@@ -512,6 +513,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
                                Filename:  blob.Name(),
                                URLPrefix: path.Dir(treeLink),
                                Metas:     ctx.Repo.Repository.ComposeDocumentMetas(),
+                               GitRepo:   ctx.Repo.GitRepo,
                        }, rd, &result)
                        if err != nil {
                                ctx.ServerError("Render", err)
@@ -570,6 +572,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
                                Filename:  blob.Name(),
                                URLPrefix: path.Dir(treeLink),
                                Metas:     ctx.Repo.Repository.ComposeDocumentMetas(),
+                               GitRepo:   ctx.Repo.GitRepo,
                        }, rd, &result)
                        if err != nil {
                                ctx.ServerError("Render", err)

diff --git a/routers/web/user/profile.go b/routers/web/user/profile.go
index e66820e1317bc9e22a21bc87b9aa5aed1f9ba93d..72d00666453e253ba1798ef78d5218c7071c5a7d 100644 (file)
--- a/routers/web/user/profile.go
+++ b/routers/web/user/profile.go
@@ -117,6 +117,7 @@ func Profile(ctx *context.Context) {
                content, err := markdown.RenderString(&markup.RenderContext{
                        URLPrefix: ctx.Repo.RepoLink,
                        Metas:     map[string]string{"mode": "document"},
+                       GitRepo:   ctx.Repo.GitRepo,
                }, ctxUser.Description)
                if err != nil {
                        ctx.ServerError("RenderString", err)