Add Goroutine stack inspector to admin/monitor (#19207)

Continues on from #19202.

Following the addition of pprof labels we can now more easily understand the relationship between a goroutine and the requests that spawn them. 

This PR takes advantage of the labels and adds a few others, then provides a mechanism for the monitoring page to query the pprof goroutine profile.

The binary profile that results from this profile is immediately piped in to the google library for parsing this and then stack traces are formed for the goroutines.

If the goroutine is within a context or has been created from a goroutine within a process context it will acquire the process description labels for that process. 

The goroutines are mapped with there associate pids and any that do not have an associated pid are placed in a group at the bottom as unbound.

In this way we should be able to more easily examine goroutines that have been stuck.

A manager command `gitea manager processes` is also provided that can export the processes (with or without stacktraces) to the command line.

Signed-off-by: Andrew Thornton <art27@cantab.net>

3 files changed, 14 insertions, 6 deletions

diff --git a/services/cron/cron.go b/services/cron/cron.go
index 9fe90d4230..ebbcd75b6d 100644
--- a/services/cron/cron.go
+++ b/services/cron/cron.go
@@ -7,9 +7,11 @@ package cron
 
 import (
 	"context"
+	"runtime/pprof"
 	"time"
 
 	"code.gitea.io/gitea/modules/graceful"
+	"code.gitea.io/gitea/modules/process"
 	"code.gitea.io/gitea/modules/sync"
 
 	"github.com/gogs/cron"
@@ -23,7 +25,9 @@ var taskStatusTable = sync.NewStatusTable()
 // NewContext begins cron tasks
 // Each cron task is run within the shutdown context as a running server
 // AtShutdown the cron server is stopped
-func NewContext() {
+func NewContext(original context.Context) {
+	defer pprof.SetGoroutineLabels(original)
+	_, _, finished := process.GetManager().AddTypedContext(graceful.GetManager().ShutdownContext(), "Service: Cron", process.SystemProcessType, true)
 	initBasicTasks()
 	initExtendedTasks()
 
@@ -42,6 +46,7 @@ func NewContext() {
 		lock.Lock()
 		started = false
 		lock.Unlock()
+		finished()
 	})
 }
 
diff --git a/services/pull/check_test.go b/services/pull/check_test.go
index 4cdd17cc7b..65bcb9c0e4 100644
--- a/services/pull/check_test.go
+++ b/services/pull/check_test.go
@@ -32,9 +32,9 @@ func TestPullRequest_AddToTaskQueue(t *testing.T) {
 		WorkerPoolConfiguration: queue.WorkerPoolConfiguration{
 			QueueLength: 10,
 			BatchLength: 1,
+			Name:        "temporary-queue",
 		},
 		Workers: 1,
-		Name:    "temporary-queue",
 	}, "")
 	assert.NoError(t, err)
 
diff --git a/services/webhook/deliver.go b/services/webhook/deliver.go
index f45e9d08d8..7998be53c2 100644
--- a/services/webhook/deliver.go
+++ b/services/webhook/deliver.go
@@ -24,6 +24,7 @@ import (
 	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/hostmatcher"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/process"
 	"code.gitea.io/gitea/modules/proxy"
 	"code.gitea.io/gitea/modules/setting"
 
@@ -31,7 +32,7 @@ import (
 )
 
 // Deliver deliver hook task
-func Deliver(t *webhook_model.HookTask) error {
+func Deliver(ctx context.Context, t *webhook_model.HookTask) error {
 	w, err := webhook_model.GetWebhookByID(t.HookID)
 	if err != nil {
 		return err
@@ -178,7 +179,7 @@ func Deliver(t *webhook_model.HookTask) error {
 		return nil
 	}
 
-	resp, err := webhookHTTPClient.Do(req.WithContext(graceful.GetManager().ShutdownContext()))
+	resp, err := webhookHTTPClient.Do(req.WithContext(ctx))
 	if err != nil {
 		t.ResponseInfo.Body = fmt.Sprintf("Delivery: %v", err)
 		return err
@@ -210,6 +211,8 @@ func DeliverHooks(ctx context.Context) {
 		return
 	default:
 	}
+	ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Service: DeliverHooks", process.SystemProcessType, true)
+	defer finished()
 	tasks, err := webhook_model.FindUndeliveredHookTasks()
 	if err != nil {
 		log.Error("DeliverHooks: %v", err)
@@ -223,7 +226,7 @@ func DeliverHooks(ctx context.Context) {
 			return
 		default:
 		}
-		if err = Deliver(t); err != nil {
+		if err = Deliver(ctx, t); err != nil {
 			log.Error("deliver: %v", err)
 		}
 	}
@@ -255,7 +258,7 @@ func DeliverHooks(ctx context.Context) {
 					return
 				default:
 				}
-				if err = Deliver(t); err != nil {
+				if err = Deliver(ctx, t); err != nil {
 					log.Error("deliver: %v", err)
 				}
 			}