Use batch database operations instead of one by one to optimze api pulls (#32680)

Resolve #31492

The response time for the Pull Requests API has improved significantly,
dropping from over `2000ms` to about `350ms` on my local machine. It's
about `6` times faster.

A key area for further optimization lies in batch-fetching data for
`apiPullRequest.ChangedFiles, apiPullRequest.Additions, and
apiPullRequest.Deletions`.

Tests `TestAPIViewPulls` does exist and new tests added.

- This PR also fixes some bugs in `GetDiff` functions.
- This PR also fixes data inconsistent in test data. For a pull request,
the head branch's reference should be equal to the reference in
`pull/xxx/head`.

1 files changed, 251 insertions, 0 deletions

diff --git a/services/convert/pull.go b/services/convert/pull.go
index 4ec24a8276..ddaaa300a4 100644
--- a/services/convert/pull.go
+++ b/services/convert/pull.go
@@ -7,9 +7,11 @@ import (
 	"context"
 	"fmt"
 
+	git_model "code.gitea.io/gitea/models/git"
 	issues_model "code.gitea.io/gitea/models/issues"
 	"code.gitea.io/gitea/models/perm"
 	access_model "code.gitea.io/gitea/models/perm/access"
+	repo_model "code.gitea.io/gitea/models/repo"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/cache"
 	"code.gitea.io/gitea/modules/git"
@@ -259,3 +261,252 @@ func ToAPIPullRequest(ctx context.Context, pr *issues_model.PullRequest, doer *u
 
 	return apiPullRequest
 }
+
+func ToAPIPullRequests(ctx context.Context, baseRepo *repo_model.Repository, prs issues_model.PullRequestList, doer *user_model.User) ([]*api.PullRequest, error) {
+	for _, pr := range prs {
+		pr.BaseRepo = baseRepo
+		if pr.BaseRepoID == pr.HeadRepoID {
+			pr.HeadRepo = baseRepo
+		}
+	}
+
+	// NOTE: load head repositories
+	if err := prs.LoadRepositories(ctx); err != nil {
+		return nil, err
+	}
+	issueList, err := prs.LoadIssues(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := issueList.LoadLabels(ctx); err != nil {
+		return nil, err
+	}
+	if err := issueList.LoadPosters(ctx); err != nil {
+		return nil, err
+	}
+	if err := issueList.LoadAttachments(ctx); err != nil {
+		return nil, err
+	}
+	if err := issueList.LoadMilestones(ctx); err != nil {
+		return nil, err
+	}
+	if err := issueList.LoadAssignees(ctx); err != nil {
+		return nil, err
+	}
+
+	reviews, err := prs.LoadReviews(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if err = reviews.LoadReviewers(ctx); err != nil {
+		return nil, err
+	}
+
+	reviewersMap := make(map[int64][]*user_model.User)
+	for _, review := range reviews {
+		if review.Reviewer != nil {
+			reviewersMap[review.IssueID] = append(reviewersMap[review.IssueID], review.Reviewer)
+		}
+	}
+
+	reviewCounts, err := prs.LoadReviewCommentsCounts(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	gitRepo, err := gitrepo.OpenRepository(ctx, baseRepo)
+	if err != nil {
+		return nil, err
+	}
+	defer gitRepo.Close()
+
+	baseRepoPerm, err := access_model.GetUserRepoPermission(ctx, baseRepo, doer)
+	if err != nil {
+		log.Error("GetUserRepoPermission[%d]: %v", baseRepo.ID, err)
+		baseRepoPerm.AccessMode = perm.AccessModeNone
+	}
+
+	apiRepo := ToRepo(ctx, baseRepo, baseRepoPerm)
+	baseBranchCache := make(map[string]*git_model.Branch)
+	apiPullRequests := make([]*api.PullRequest, 0, len(prs))
+	for _, pr := range prs {
+		apiIssue := ToAPIIssue(ctx, doer, pr.Issue)
+
+		apiPullRequest := &api.PullRequest{
+			ID:             pr.ID,
+			URL:            pr.Issue.HTMLURL(),
+			Index:          pr.Index,
+			Poster:         apiIssue.Poster,
+			Title:          apiIssue.Title,
+			Body:           apiIssue.Body,
+			Labels:         apiIssue.Labels,
+			Milestone:      apiIssue.Milestone,
+			Assignee:       apiIssue.Assignee,
+			Assignees:      apiIssue.Assignees,
+			State:          apiIssue.State,
+			Draft:          pr.IsWorkInProgress(ctx),
+			IsLocked:       apiIssue.IsLocked,
+			Comments:       apiIssue.Comments,
+			ReviewComments: reviewCounts[pr.IssueID],
+			HTMLURL:        pr.Issue.HTMLURL(),
+			DiffURL:        pr.Issue.DiffURL(),
+			PatchURL:       pr.Issue.PatchURL(),
+			HasMerged:      pr.HasMerged,
+			MergeBase:      pr.MergeBase,
+			Mergeable:      pr.Mergeable(ctx),
+			Deadline:       apiIssue.Deadline,
+			Created:        pr.Issue.CreatedUnix.AsTimePtr(),
+			Updated:        pr.Issue.UpdatedUnix.AsTimePtr(),
+			PinOrder:       apiIssue.PinOrder,
+
+			AllowMaintainerEdit: pr.AllowMaintainerEdit,
+
+			Base: &api.PRBranchInfo{
+				Name:       pr.BaseBranch,
+				Ref:        pr.BaseBranch,
+				RepoID:     pr.BaseRepoID,
+				Repository: apiRepo,
+			},
+			Head: &api.PRBranchInfo{
+				Name:   pr.HeadBranch,
+				Ref:    fmt.Sprintf("%s%d/head", git.PullPrefix, pr.Index),
+				RepoID: -1,
+			},
+		}
+
+		pr.RequestedReviewers = reviewersMap[pr.IssueID]
+		for _, reviewer := range pr.RequestedReviewers {
+			apiPullRequest.RequestedReviewers = append(apiPullRequest.RequestedReviewers, ToUser(ctx, reviewer, nil))
+		}
+
+		for _, reviewerTeam := range pr.RequestedReviewersTeams {
+			convertedTeam, err := ToTeam(ctx, reviewerTeam, true)
+			if err != nil {
+				log.Error("LoadRequestedReviewersTeams[%d]: %v", pr.ID, err)
+				return nil, err
+			}
+
+			apiPullRequest.RequestedReviewersTeams = append(apiPullRequest.RequestedReviewersTeams, convertedTeam)
+		}
+
+		if pr.Issue.ClosedUnix != 0 {
+			apiPullRequest.Closed = pr.Issue.ClosedUnix.AsTimePtr()
+		}
+
+		baseBranch, ok := baseBranchCache[pr.BaseBranch]
+		if !ok {
+			baseBranch, err = git_model.GetBranch(ctx, baseRepo.ID, pr.BaseBranch)
+			if err == nil {
+				baseBranchCache[pr.BaseBranch] = baseBranch
+			} else if !git_model.IsErrBranchNotExist(err) {
+				return nil, err
+			}
+		}
+
+		if baseBranch != nil {
+			apiPullRequest.Base.Sha = baseBranch.CommitID
+		}
+
+		if pr.Flow == issues_model.PullRequestFlowAGit {
+			apiPullRequest.Head.Sha, err = gitRepo.GetRefCommitID(pr.GetGitRefName())
+			if err != nil {
+				log.Error("GetRefCommitID[%s]: %v", pr.GetGitRefName(), err)
+				return nil, err
+			}
+			apiPullRequest.Head.RepoID = pr.BaseRepoID
+			apiPullRequest.Head.Repository = apiPullRequest.Base.Repository
+			apiPullRequest.Head.Name = ""
+		}
+
+		var headGitRepo *git.Repository
+		if pr.HeadRepo != nil && pr.Flow == issues_model.PullRequestFlowGithub {
+			if pr.HeadRepoID == pr.BaseRepoID {
+				apiPullRequest.Head.RepoID = pr.HeadRepo.ID
+				apiPullRequest.Head.Repository = apiRepo
+				headGitRepo = gitRepo
+			} else {
+				p, err := access_model.GetUserRepoPermission(ctx, pr.HeadRepo, doer)
+				if err != nil {
+					log.Error("GetUserRepoPermission[%d]: %v", pr.HeadRepoID, err)
+					p.AccessMode = perm.AccessModeNone
+				}
+
+				apiPullRequest.Head.RepoID = pr.HeadRepo.ID
+				apiPullRequest.Head.Repository = ToRepo(ctx, pr.HeadRepo, p)
+
+				headGitRepo, err = gitrepo.OpenRepository(ctx, pr.HeadRepo)
+				if err != nil {
+					log.Error("OpenRepository[%s]: %v", pr.HeadRepo.RepoPath(), err)
+					return nil, err
+				}
+				defer headGitRepo.Close()
+			}
+
+			headBranch, err := headGitRepo.GetBranch(pr.HeadBranch)
+			if err != nil && !git.IsErrBranchNotExist(err) {
+				log.Error("GetBranch[%s]: %v", pr.HeadBranch, err)
+				return nil, err
+			}
+
+			// Outer scope variables to be used in diff calculation
+			var (
+				startCommitID string
+				endCommitID   string
+			)
+
+			if git.IsErrBranchNotExist(err) {
+				headCommitID, err := headGitRepo.GetRefCommitID(apiPullRequest.Head.Ref)
+				if err != nil && !git.IsErrNotExist(err) {
+					log.Error("GetCommit[%s]: %v", pr.HeadBranch, err)
+					return nil, err
+				}
+				if err == nil {
+					apiPullRequest.Head.Sha = headCommitID
+					endCommitID = headCommitID
+				}
+			} else {
+				commit, err := headBranch.GetCommit()
+				if err != nil && !git.IsErrNotExist(err) {
+					log.Error("GetCommit[%s]: %v", headBranch.Name, err)
+					return nil, err
+				}
+				if err == nil {
+					apiPullRequest.Head.Ref = pr.HeadBranch
+					apiPullRequest.Head.Sha = commit.ID.String()
+					endCommitID = commit.ID.String()
+				}
+			}
+
+			// Calculate diff
+			startCommitID = pr.MergeBase
+
+			apiPullRequest.ChangedFiles, apiPullRequest.Additions, apiPullRequest.Deletions, err = gitRepo.GetDiffShortStat(startCommitID, endCommitID)
+			if err != nil {
+				log.Error("GetDiffShortStat: %v", err)
+			}
+		}
+
+		if len(apiPullRequest.Head.Sha) == 0 && len(apiPullRequest.Head.Ref) != 0 {
+			refs, err := gitRepo.GetRefsFiltered(apiPullRequest.Head.Ref)
+			if err != nil {
+				log.Error("GetRefsFiltered[%s]: %v", apiPullRequest.Head.Ref, err)
+				return nil, err
+			} else if len(refs) == 0 {
+				log.Error("unable to resolve PR head ref")
+			} else {
+				apiPullRequest.Head.Sha = refs[0].Object.String()
+			}
+		}
+
+		if pr.HasMerged {
+			apiPullRequest.Merged = pr.MergedUnix.AsTimePtr()
+			apiPullRequest.MergedCommitID = &pr.MergedCommitID
+			apiPullRequest.MergedBy = ToUser(ctx, pr.Merger, nil)
+		}
+
+		apiPullRequests = append(apiPullRequests, apiPullRequest)
+	}
+
+	return apiPullRequests, nil
+}