Add Goroutine stack inspector to admin/monitor (#19207)

Continues on from #19202.

Following the addition of pprof labels we can now more easily understand the relationship between a goroutine and the requests that spawn them. 

This PR takes advantage of the labels and adds a few others, then provides a mechanism for the monitoring page to query the pprof goroutine profile.

The binary profile that results from this profile is immediately piped in to the google library for parsing this and then stack traces are formed for the goroutines.

If the goroutine is within a context or has been created from a goroutine within a process context it will acquire the process description labels for that process. 

The goroutines are mapped with there associate pids and any that do not have an associated pid are placed in a group at the bottom as unbound.

In this way we should be able to more easily examine goroutines that have been stuck.

A manager command `gitea manager processes` is also provided that can export the processes (with or without stacktraces) to the command line.

Signed-off-by: Andrew Thornton <art27@cantab.net>

6 files changed, 192 insertions, 7 deletions

diff --git a/routers/common/middleware.go b/routers/common/middleware.go
index 591c4cf30e..6ea1e1dfbe 100644
--- a/routers/common/middleware.go
+++ b/routers/common/middleware.go
@@ -27,7 +27,7 @@ func Middlewares() []func(http.Handler) http.Handler {
 				// First of all escape the URL RawPath to ensure that all routing is done using a correctly escaped URL
 				req.URL.RawPath = req.URL.EscapedPath()
 
-				ctx, _, finished := process.GetManager().AddContext(req.Context(), fmt.Sprintf("%s: %s", req.Method, req.RequestURI))
+				ctx, _, finished := process.GetManager().AddTypedContext(req.Context(), fmt.Sprintf("%s: %s", req.Method, req.RequestURI), process.RequestProcessType, true)
 				defer finished()
 				next.ServeHTTP(context.NewResponse(resp), req.WithContext(ctx))
 			})
diff --git a/routers/init.go b/routers/init.go
index 4899947897..62a9e4002b 100644
--- a/routers/init.go
+++ b/routers/init.go
@@ -141,7 +141,7 @@ func GlobalInitInstalled(ctx context.Context) {
 	models.NewRepoContext()
 
 	// Booting long running goroutines.
-	cron.NewContext()
+	cron.NewContext(ctx)
 	issue_indexer.InitIssueIndexer(false)
 	code_indexer.Init()
 	mustInit(stats_indexer.Init)
diff --git a/routers/private/internal.go b/routers/private/internal.go
index 263180bd58..6ba87d67bf 100644
--- a/routers/private/internal.go
+++ b/routers/private/internal.go
@@ -70,6 +70,7 @@ func Routes() *web.Route {
 	r.Post("/manager/release-and-reopen-logging", ReleaseReopenLogging)
 	r.Post("/manager/add-logger", bind(private.LoggerOptions{}), AddLogger)
 	r.Post("/manager/remove-logger/{group}/{name}", RemoveLogger)
+	r.Get("/manager/processes", Processes)
 	r.Post("/mail/send", SendEmail)
 	r.Post("/restore_repo", RestoreRepo)
 
diff --git a/routers/private/manager_process.go b/routers/private/manager_process.go
new file mode 100644
index 0000000000..f8932d61fa
--- /dev/null
+++ b/routers/private/manager_process.go
@@ -0,0 +1,161 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package private
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"runtime"
+	"time"
+
+	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/private"
+	process_module "code.gitea.io/gitea/modules/process"
+)
+
+// Processes prints out the processes
+func Processes(ctx *context.PrivateContext) {
+	pid := ctx.FormString("cancel-pid")
+	if pid != "" {
+		process_module.GetManager().Cancel(process_module.IDType(pid))
+		runtime.Gosched()
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	flat := ctx.FormBool("flat")
+	noSystem := ctx.FormBool("no-system")
+	stacktraces := ctx.FormBool("stacktraces")
+	json := ctx.FormBool("json")
+
+	var processes []*process_module.Process
+	goroutineCount := int64(0)
+	processCount := 0
+	var err error
+	if stacktraces {
+		processes, processCount, goroutineCount, err = process_module.GetManager().ProcessStacktraces(flat, noSystem)
+		if err != nil {
+			log.Error("Unable to get stacktrace: %v", err)
+			ctx.JSON(http.StatusInternalServerError, private.Response{
+				Err: fmt.Sprintf("Failed to get stacktraces: %v", err),
+			})
+			return
+		}
+	} else {
+		processes, processCount = process_module.GetManager().Processes(flat, noSystem)
+	}
+
+	if json {
+		ctx.JSON(http.StatusOK, map[string]interface{}{
+			"TotalNumberOfGoroutines": goroutineCount,
+			"TotalNumberOfProcesses":  processCount,
+			"Processes":               processes,
+		})
+		return
+	}
+
+	ctx.Resp.Header().Set("Content-Type", "text/plain;charset=utf-8")
+	ctx.Resp.WriteHeader(http.StatusOK)
+
+	if err := writeProcesses(ctx.Resp, processes, processCount, goroutineCount, "", flat); err != nil {
+		log.Error("Unable to write out process stacktrace: %v", err)
+		if !ctx.Written() {
+			ctx.JSON(http.StatusInternalServerError, private.Response{
+				Err: fmt.Sprintf("Failed to get stacktraces: %v", err),
+			})
+		}
+		return
+	}
+}
+
+func writeProcesses(out io.Writer, processes []*process_module.Process, processCount int, goroutineCount int64, indent string, flat bool) error {
+	if goroutineCount > 0 {
+		if _, err := fmt.Fprintf(out, "%sTotal Number of Goroutines: %d\n", indent, goroutineCount); err != nil {
+			return err
+		}
+	}
+	if _, err := fmt.Fprintf(out, "%sTotal Number of Processes: %d\n", indent, processCount); err != nil {
+		return err
+	}
+	if len(processes) > 0 {
+		if err := writeProcess(out, processes[0], "  ", flat); err != nil {
+			return err
+		}
+	}
+	if len(processes) > 1 {
+		for _, process := range processes[1:] {
+			if _, err := fmt.Fprintf(out, "%s  | \n", indent); err != nil {
+				return err
+			}
+			if err := writeProcess(out, process, "  ", flat); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func writeProcess(out io.Writer, process *process_module.Process, indent string, flat bool) error {
+	sb := &bytes.Buffer{}
+	if flat {
+		if process.ParentPID != "" {
+			_, _ = fmt.Fprintf(sb, "%s+ PID: %s\t\tType: %s\n", indent, process.PID, process.Type)
+		} else {
+			_, _ = fmt.Fprintf(sb, "%s+ PID: %s:%s\tType: %s\n", indent, process.ParentPID, process.PID, process.Type)
+		}
+	} else {
+		_, _ = fmt.Fprintf(sb, "%s+ PID: %s\tType: %s\n", indent, process.PID, process.Type)
+	}
+	indent += "| "
+
+	_, _ = fmt.Fprintf(sb, "%sDescription: %s\n", indent, process.Description)
+	_, _ = fmt.Fprintf(sb, "%sStart:       %s\n", indent, process.Start)
+
+	if len(process.Stacks) > 0 {
+		_, _ = fmt.Fprintf(sb, "%sGoroutines:\n", indent)
+		for _, stack := range process.Stacks {
+			indent := indent + "  "
+			_, _ = fmt.Fprintf(sb, "%s+ Description: %s", indent, stack.Description)
+			if stack.Count > 1 {
+				_, _ = fmt.Fprintf(sb, "* %d", stack.Count)
+			}
+			_, _ = fmt.Fprintf(sb, "\n")
+			indent += "| "
+			if len(stack.Labels) > 0 {
+				_, _ = fmt.Fprintf(sb, "%sLabels:      %q:%q", indent, stack.Labels[0].Name, stack.Labels[0].Value)
+
+				if len(stack.Labels) > 1 {
+					for _, label := range stack.Labels[1:] {
+						_, _ = fmt.Fprintf(sb, ", %q:%q", label.Name, label.Value)
+					}
+				}
+				_, _ = fmt.Fprintf(sb, "\n")
+			}
+			_, _ = fmt.Fprintf(sb, "%sStack:\n", indent)
+			indent += "  "
+			for _, entry := range stack.Entry {
+				_, _ = fmt.Fprintf(sb, "%s+ %s\n", indent, entry.Function)
+				_, _ = fmt.Fprintf(sb, "%s| %s:%d\n", indent, entry.File, entry.Line)
+			}
+		}
+	}
+	if _, err := out.Write(sb.Bytes()); err != nil {
+		return err
+	}
+	sb.Reset()
+	if len(process.Children) > 0 {
+		if _, err := fmt.Fprintf(out, "%sChildren:\n", indent); err != nil {
+			return err
+		}
+		for _, child := range process.Children {
+			if err := writeProcess(out, child, indent+"  ", flat); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
diff --git a/routers/web/admin/admin.go b/routers/web/admin/admin.go
index 4c700df354..d4093f2049 100644
--- a/routers/web/admin/admin.go
+++ b/routers/web/admin/admin.go
@@ -35,10 +35,11 @@ import (
 )
 
 const (
-	tplDashboard base.TplName = "admin/dashboard"
-	tplConfig    base.TplName = "admin/config"
-	tplMonitor   base.TplName = "admin/monitor"
-	tplQueue     base.TplName = "admin/queue"
+	tplDashboard  base.TplName = "admin/dashboard"
+	tplConfig     base.TplName = "admin/config"
+	tplMonitor    base.TplName = "admin/monitor"
+	tplStacktrace base.TplName = "admin/stacktrace"
+	tplQueue      base.TplName = "admin/queue"
 )
 
 var sysStatus struct {
@@ -326,12 +327,33 @@ func Monitor(ctx *context.Context) {
 	ctx.Data["Title"] = ctx.Tr("admin.monitor")
 	ctx.Data["PageIsAdmin"] = true
 	ctx.Data["PageIsAdminMonitor"] = true
-	ctx.Data["Processes"] = process.GetManager().Processes(true)
+	ctx.Data["Processes"], ctx.Data["ProcessCount"] = process.GetManager().Processes(false, true)
 	ctx.Data["Entries"] = cron.ListTasks()
 	ctx.Data["Queues"] = queue.GetManager().ManagedQueues()
+
 	ctx.HTML(http.StatusOK, tplMonitor)
 }
 
+// GoroutineStacktrace show admin monitor goroutines page
+func GoroutineStacktrace(ctx *context.Context) {
+	ctx.Data["Title"] = ctx.Tr("admin.monitor")
+	ctx.Data["PageIsAdmin"] = true
+	ctx.Data["PageIsAdminMonitor"] = true
+
+	processStacks, processCount, goroutineCount, err := process.GetManager().ProcessStacktraces(false, false)
+	if err != nil {
+		ctx.ServerError("GoroutineStacktrace", err)
+		return
+	}
+
+	ctx.Data["ProcessStacks"] = processStacks
+
+	ctx.Data["GoroutineCount"] = goroutineCount
+	ctx.Data["ProcessCount"] = processCount
+
+	ctx.HTML(http.StatusOK, tplStacktrace)
+}
+
 // MonitorCancel cancels a process
 func MonitorCancel(ctx *context.Context) {
 	pid := ctx.Params("pid")
diff --git a/routers/web/web.go b/routers/web/web.go
index 60e104ccf8..5ff8174be4 100644
--- a/routers/web/web.go
+++ b/routers/web/web.go
@@ -436,6 +436,7 @@ func RegisterRoutes(m *web.Route) {
 		m.Post("/config/test_mail", admin.SendTestMail)
 		m.Group("/monitor", func() {
 			m.Get("", admin.Monitor)
+			m.Get("/stacktrace", admin.GoroutineStacktrace)
 			m.Post("/cancel/{pid}", admin.MonitorCancel)
 			m.Group("/queue/{qid}", func() {
 				m.Get("", admin.Queue)