Graceful: Cancel Process on monitor pages & HammerTime (#9213)

* Graceful: Create callbacks to with contexts

* Graceful: Say when Gitea is completely finished

* Graceful: Git and Process within HammerTime

Force all git commands to terminate at HammerTime
Force all process commands to terminate at HammerTime
Move almost all git processes to run as git Commands

* Graceful: Always Hammer after Shutdown

* ProcessManager: Add cancel functionality

* Fix tests

* Make sure that process.Manager.Kill() cancels

* Make threadsafe access to Processes and remove own unused Kill

* Remove cmd from the process manager as it is no longer used

* the default context is the correct context

* get rid of double till

3 files changed, 65 insertions, 72 deletions

diff --git a/models/pull.go b/models/pull.go
index 174faee97a..5b18c9ed10 100644
--- a/models/pull.go
+++ b/models/pull.go
@@ -18,7 +18,6 @@ import (
 
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/process"
 	"code.gitea.io/gitea/modules/setting"
 	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/sync"
@@ -536,16 +535,13 @@ func (pr *PullRequest) getMergeCommit() (*git.Commit, error) {
 	headFile := pr.GetGitRefName()
 
 	// Check if a pull request is merged into BaseBranch
-	_, stderr, err := process.GetManager().ExecDirEnv(-1, "", fmt.Sprintf("isMerged (git merge-base --is-ancestor): %d", pr.BaseRepo.ID),
-		[]string{"GIT_INDEX_FILE=" + indexTmpPath, "GIT_DIR=" + pr.BaseRepo.RepoPath()},
-		git.GitExecutable, "merge-base", "--is-ancestor", headFile, pr.BaseBranch)
-
+	_, err := git.NewCommand("merge-base", "--is-ancestor", headFile, pr.BaseBranch).RunInDirWithEnv(pr.BaseRepo.RepoPath(), []string{"GIT_INDEX_FILE=" + indexTmpPath, "GIT_DIR=" + pr.BaseRepo.RepoPath()})
 	if err != nil {
 		// Errors are signaled by a non-zero status that is not 1
 		if strings.Contains(err.Error(), "exit status 1") {
 			return nil, nil
 		}
-		return nil, fmt.Errorf("git merge-base --is-ancestor: %v %v", stderr, err)
+		return nil, fmt.Errorf("git merge-base --is-ancestor: %v", err)
 	}
 
 	commitIDBytes, err := ioutil.ReadFile(pr.BaseRepo.RepoPath() + "/" + headFile)
@@ -559,11 +555,9 @@ func (pr *PullRequest) getMergeCommit() (*git.Commit, error) {
 	cmd := commitID[:40] + ".." + pr.BaseBranch
 
 	// Get the commit from BaseBranch where the pull request got merged
-	mergeCommit, stderr, err := process.GetManager().ExecDirEnv(-1, "", fmt.Sprintf("isMerged (git rev-list --ancestry-path --merges --reverse): %d", pr.BaseRepo.ID),
-		[]string{"GIT_INDEX_FILE=" + indexTmpPath, "GIT_DIR=" + pr.BaseRepo.RepoPath()},
-		git.GitExecutable, "rev-list", "--ancestry-path", "--merges", "--reverse", cmd)
+	mergeCommit, err := git.NewCommand("rev-list", "--ancestry-path", "--merges", "--reverse", cmd).RunInDirWithEnv("", []string{"GIT_INDEX_FILE=" + indexTmpPath, "GIT_DIR=" + pr.BaseRepo.RepoPath()})
 	if err != nil {
-		return nil, fmt.Errorf("git rev-list --ancestry-path --merges --reverse: %v %v", stderr, err)
+		return nil, fmt.Errorf("git rev-list --ancestry-path --merges --reverse: %v", err)
 	} else if len(mergeCommit) < 40 {
 		// PR was fast-forwarded, so just use last commit of PR
 		mergeCommit = commitID[:40]
@@ -621,12 +615,9 @@ func (pr *PullRequest) testPatch(e Engine) (err error) {
 	indexTmpPath := filepath.Join(os.TempDir(), "gitea-"+pr.BaseRepo.Name+"-"+strconv.Itoa(time.Now().Nanosecond()))
 	defer os.Remove(indexTmpPath)
 
-	var stderr string
-	_, stderr, err = process.GetManager().ExecDirEnv(-1, "", fmt.Sprintf("testPatch (git read-tree): %d", pr.BaseRepo.ID),
-		[]string{"GIT_DIR=" + pr.BaseRepo.RepoPath(), "GIT_INDEX_FILE=" + indexTmpPath},
-		git.GitExecutable, "read-tree", pr.BaseBranch)
+	_, err = git.NewCommand("read-tree", pr.BaseBranch).RunInDirWithEnv("", []string{"GIT_DIR=" + pr.BaseRepo.RepoPath(), "GIT_INDEX_FILE=" + indexTmpPath})
 	if err != nil {
-		return fmt.Errorf("git read-tree --index-output=%s %s: %v - %s", indexTmpPath, pr.BaseBranch, err, stderr)
+		return fmt.Errorf("git read-tree --index-output=%s %s: %v", indexTmpPath, pr.BaseBranch, err)
 	}
 
 	prUnit, err := pr.BaseRepo.getUnit(e, UnitTypePullRequests)
@@ -642,9 +633,15 @@ func (pr *PullRequest) testPatch(e Engine) (err error) {
 	args = append(args, patchPath)
 	pr.ConflictedFiles = []string{}
 
-	_, stderr, err = process.GetManager().ExecDirEnv(-1, "", fmt.Sprintf("testPatch (git apply --check): %d", pr.BaseRepo.ID),
+	stderrBuilder := new(strings.Builder)
+	err = git.NewCommand(args...).RunInDirTimeoutEnvPipeline(
 		[]string{"GIT_INDEX_FILE=" + indexTmpPath, "GIT_DIR=" + pr.BaseRepo.RepoPath()},
-		git.GitExecutable, args...)
+		-1,
+		"",
+		nil,
+		stderrBuilder)
+	stderr := stderrBuilder.String()
+
 	if err != nil {
 		for i := range patchConflicts {
 			if strings.Contains(stderr, patchConflicts[i]) {
diff --git a/models/repo.go b/models/repo.go
index 0ccf786db3..eec9065359 100644
--- a/models/repo.go
+++ b/models/repo.go
@@ -30,7 +30,6 @@ import (
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/options"
-	"code.gitea.io/gitea/modules/process"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/structs"
 	api "code.gitea.io/gitea/modules/structs"
@@ -1202,11 +1201,11 @@ func initRepoCommit(tmpPath string, u *User) (err error) {
 		"GIT_COMMITTER_DATE="+commitTimeStr,
 	)
 
-	var stderr string
-	if _, stderr, err = process.GetManager().ExecDir(-1,
-		tmpPath, fmt.Sprintf("initRepoCommit (git add): %s", tmpPath),
-		git.GitExecutable, "add", "--all"); err != nil {
-		return fmt.Errorf("git add: %s", stderr)
+	if stdout, err := git.NewCommand("add", "--all").
+		SetDescription(fmt.Sprintf("initRepoCommit (git add): %s", tmpPath)).
+		RunInDir(tmpPath); err != nil {
+		log.Error("git add --all failed: Stdout: %s\nError: %v", stdout, err)
+		return fmt.Errorf("git add --all: %v", err)
 	}
 
 	binVersion, err := git.BinVersion()
@@ -1228,18 +1227,20 @@ func initRepoCommit(tmpPath string, u *User) (err error) {
 		}
 	}
 
-	if _, stderr, err = process.GetManager().ExecDirEnv(-1,
-		tmpPath, fmt.Sprintf("initRepoCommit (git commit): %s", tmpPath),
-		env,
-		git.GitExecutable, args...); err != nil {
-		return fmt.Errorf("git commit: %s", stderr)
+	if stdout, err := git.NewCommand(args...).
+		SetDescription(fmt.Sprintf("initRepoCommit (git commit): %s", tmpPath)).
+		RunInDirWithEnv(tmpPath, env); err != nil {
+		log.Error("Failed to commit: %v: Stdout: %s\nError: %v", args, stdout, err)
+		return fmt.Errorf("git commit: %v", err)
 	}
 
-	if _, stderr, err = process.GetManager().ExecDir(-1,
-		tmpPath, fmt.Sprintf("initRepoCommit (git push): %s", tmpPath),
-		git.GitExecutable, "push", "origin", "master"); err != nil {
-		return fmt.Errorf("git push: %s", stderr)
+	if stdout, err := git.NewCommand("push", "origin", "master").
+		SetDescription(fmt.Sprintf("initRepoCommit (git push): %s", tmpPath)).
+		RunInDir(tmpPath); err != nil {
+		log.Error("Failed to push back to master: Stdout: %s\nError: %v", stdout, err)
+		return fmt.Errorf("git push: %v", err)
 	}
+
 	return nil
 }
 
@@ -1297,14 +1298,11 @@ func prepareRepoCommit(e Engine, repo *Repository, tmpDir, repoPath string, opts
 	)
 
 	// Clone to temporary path and do the init commit.
-	_, stderr, err := process.GetManager().ExecDirEnv(
-		-1, "",
-		fmt.Sprintf("initRepository(git clone): %s", repoPath),
-		env,
-		git.GitExecutable, "clone", repoPath, tmpDir,
-	)
-	if err != nil {
-		return fmt.Errorf("git clone: %v - %s", err, stderr)
+	if stdout, err := git.NewCommand("clone", repoPath, tmpDir).
+		SetDescription(fmt.Sprintf("initRepository (git clone): %s to %s", repoPath, tmpDir)).
+		RunInDirWithEnv("", env); err != nil {
+		log.Error("Failed to clone from %v into %s: stdout: %s\nError: %v", repo, tmpDir, stdout, err)
+		return fmt.Errorf("git clone: %v", err)
 	}
 
 	// README
@@ -1584,11 +1582,11 @@ func CreateRepository(doer, u *User, opts CreateRepoOptions) (_ *Repository, err
 			}
 		}
 
-		_, stderr, err := process.GetManager().ExecDir(-1,
-			repoPath, fmt.Sprintf("CreateRepository(git update-server-info): %s", repoPath),
-			git.GitExecutable, "update-server-info")
-		if err != nil {
-			return nil, errors.New("CreateRepository(git update-server-info): " + stderr)
+		if stdout, err := git.NewCommand("update-server-info").
+			SetDescription(fmt.Sprintf("CreateRepository(git update-server-info): %s", repoPath)).
+			RunInDir(repoPath); err != nil {
+			log.Error("CreateRepitory(git update-server-info) in %v: Stdout: %s\nError: %v", repo, stdout, err)
+			return nil, fmt.Errorf("CreateRepository(git update-server-info): %v", err)
 		}
 	}
 
@@ -2422,12 +2420,13 @@ func GitGcRepos() error {
 				if err := repo.GetOwner(); err != nil {
 					return err
 				}
-				_, stderr, err := process.GetManager().ExecDir(
-					time.Duration(setting.Git.Timeout.GC)*time.Second,
-					RepoPath(repo.Owner.Name, repo.Name), "Repository garbage collection",
-					git.GitExecutable, args...)
-				if err != nil {
-					return fmt.Errorf("%v: %v", err, stderr)
+				if stdout, err := git.NewCommand(args...).
+					SetDescription(fmt.Sprintf("Repository Garbage Collection: %s", repo.FullName())).
+					RunInDirTimeout(
+						time.Duration(setting.Git.Timeout.GC)*time.Second,
+						RepoPath(repo.Owner.Name, repo.Name)); err != nil {
+					log.Error("Repository garbage collection failed for %v. Stdout: %s\nError: %v", repo, stdout, err)
+					return fmt.Errorf("Repository garbage collection failed: Error: %v", err)
 				}
 				return nil
 			})
@@ -2647,18 +2646,19 @@ func ForkRepository(doer, owner *User, oldRepo *Repository, name, desc string) (
 	}
 
 	repoPath := RepoPath(owner.Name, repo.Name)
-	_, stderr, err := process.GetManager().ExecTimeout(10*time.Minute,
-		fmt.Sprintf("ForkRepository(git clone): %s/%s", owner.Name, repo.Name),
-		git.GitExecutable, "clone", "--bare", oldRepo.repoPath(sess), repoPath)
-	if err != nil {
-		return nil, fmt.Errorf("git clone: %v", stderr)
-	}
-
-	_, stderr, err = process.GetManager().ExecDir(-1,
-		repoPath, fmt.Sprintf("ForkRepository(git update-server-info): %s", repoPath),
-		git.GitExecutable, "update-server-info")
-	if err != nil {
-		return nil, fmt.Errorf("git update-server-info: %v", stderr)
+	if stdout, err := git.NewCommand(
+		"clone", "--bare", oldRepo.repoPath(sess), repoPath).
+		SetDescription(fmt.Sprintf("ForkRepository(git clone): %s to %s", oldRepo.FullName(), repo.FullName())).
+		RunInDirTimeout(10*time.Minute, ""); err != nil {
+		log.Error("Fork Repository (git clone) Failed for %v (from %v):\nStdout: %s\nError: %v", repo, oldRepo, stdout, err)
+		return nil, fmt.Errorf("git clone: %v", err)
+	}
+
+	if stdout, err := git.NewCommand("update-server-info").
+		SetDescription(fmt.Sprintf("ForkRepository(git update-server-info): %s", repo.FullName())).
+		RunInDir(repoPath); err != nil {
+		log.Error("Fork Repository (git update-server-info) failed for %v:\nStdout: %s\nError: %v", repo, stdout, err)
+		return nil, fmt.Errorf("git update-server-info: %v", err)
 	}
 
 	if err = createDelegateHooks(repoPath); err != nil {
diff --git a/models/repo_generate.go b/models/repo_generate.go
index 56a3940ac1..6dd8540d9e 100644
--- a/models/repo_generate.go
+++ b/models/repo_generate.go
@@ -16,7 +16,6 @@ import (
 
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/process"
 	"code.gitea.io/gitea/modules/util"
 
 	"github.com/gobwas/glob"
@@ -168,14 +167,11 @@ func generateRepoCommit(e Engine, repo, templateRepo, generateRepo *Repository,
 	}
 
 	repoPath := repo.repoPath(e)
-	_, stderr, err := process.GetManager().ExecDirEnv(
-		-1, tmpDir,
-		fmt.Sprintf("generateRepoCommit(git remote add): %s", repoPath),
-		env,
-		git.GitExecutable, "remote", "add", "origin", repoPath,
-	)
-	if err != nil {
-		return fmt.Errorf("git remote add: %v - %s", err, stderr)
+	if stdout, err := git.NewCommand("remote", "add", "origin", repoPath).
+		SetDescription(fmt.Sprintf("generateRepoCommit (git remote add): %s to %s", templateRepoPath, tmpDir)).
+		RunInDirWithEnv(tmpDir, env); err != nil {
+		log.Error("Unable to add %v as remote origin to temporary repo to %s: stdout %s\nError: %v", repo, tmpDir, stdout, err)
+		return fmt.Errorf("git remote add: %v", err)
 	}
 
 	return initRepoCommit(tmpDir, repo.Owner)