Add Goroutine stack inspector to admin/monitor (#19207)

Continues on from #19202.

Following the addition of pprof labels we can now more easily understand the relationship between a goroutine and the requests that spawn them. 

This PR takes advantage of the labels and adds a few others, then provides a mechanism for the monitoring page to query the pprof goroutine profile.

The binary profile that results from this profile is immediately piped in to the google library for parsing this and then stack traces are formed for the goroutines.

If the goroutine is within a context or has been created from a goroutine within a process context it will acquire the process description labels for that process. 

The goroutines are mapped with there associate pids and any that do not have an associated pid are placed in a group at the bottom as unbound.

In this way we should be able to more easily examine goroutines that have been stuck.

A manager command `gitea manager processes` is also provided that can export the processes (with or without stacktraces) to the command line.

Signed-off-by: Andrew Thornton <art27@cantab.net>

4 files changed, 424 insertions, 349 deletions

diff --git a/cmd/manager.go b/cmd/manager.go
index 50b66cc7f2..03fe23aa9e 100644
--- a/cmd/manager.go
+++ b/cmd/manager.go
@@ -10,7 +10,6 @@ import (
 	"os"
 	"time"
 
-	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/private"
 
 	"github.com/urfave/cli"
@@ -27,6 +26,7 @@ var (
 			subcmdRestart,
 			subcmdFlushQueues,
 			subcmdLogging,
+			subCmdProcesses,
 		},
 	}
 	subcmdShutdown = cli.Command{
@@ -68,326 +68,38 @@ var (
 			},
 		},
 	}
-	defaultLoggingFlags = []cli.Flag{
-		cli.StringFlag{
-			Name:  "group, g",
-			Usage: "Group to add logger to - will default to \"default\"",
-		}, cli.StringFlag{
-			Name:  "name, n",
-			Usage: "Name of the new logger - will default to mode",
-		}, cli.StringFlag{
-			Name:  "level, l",
-			Usage: "Logging level for the new logger",
-		}, cli.StringFlag{
-			Name:  "stacktrace-level, L",
-			Usage: "Stacktrace logging level",
-		}, cli.StringFlag{
-			Name:  "flags, F",
-			Usage: "Flags for the logger",
-		}, cli.StringFlag{
-			Name:  "expression, e",
-			Usage: "Matching expression for the logger",
-		}, cli.StringFlag{
-			Name:  "prefix, p",
-			Usage: "Prefix for the logger",
-		}, cli.BoolFlag{
-			Name:  "color",
-			Usage: "Use color in the logs",
-		}, cli.BoolFlag{
-			Name: "debug",
-		},
-	}
-	subcmdLogging = cli.Command{
-		Name:  "logging",
-		Usage: "Adjust logging commands",
-		Subcommands: []cli.Command{
-			{
-				Name:  "pause",
-				Usage: "Pause logging (Gitea will buffer logs up to a certain point and will drop them after that point)",
-				Flags: []cli.Flag{
-					cli.BoolFlag{
-						Name: "debug",
-					},
-				},
-				Action: runPauseLogging,
-			}, {
-				Name:  "resume",
-				Usage: "Resume logging",
-				Flags: []cli.Flag{
-					cli.BoolFlag{
-						Name: "debug",
-					},
-				},
-				Action: runResumeLogging,
-			}, {
-				Name:  "release-and-reopen",
-				Usage: "Cause Gitea to release and re-open files used for logging",
-				Flags: []cli.Flag{
-					cli.BoolFlag{
-						Name: "debug",
-					},
-				},
-				Action: runReleaseReopenLogging,
-			}, {
-				Name:      "remove",
-				Usage:     "Remove a logger",
-				ArgsUsage: "[name] Name of logger to remove",
-				Flags: []cli.Flag{
-					cli.BoolFlag{
-						Name: "debug",
-					}, cli.StringFlag{
-						Name:  "group, g",
-						Usage: "Group to add logger to - will default to \"default\"",
-					},
-				},
-				Action: runRemoveLogger,
-			}, {
-				Name:  "add",
-				Usage: "Add a logger",
-				Subcommands: []cli.Command{
-					{
-						Name:  "console",
-						Usage: "Add a console logger",
-						Flags: append(defaultLoggingFlags,
-							cli.BoolFlag{
-								Name:  "stderr",
-								Usage: "Output console logs to stderr - only relevant for console",
-							}),
-						Action: runAddConsoleLogger,
-					}, {
-						Name:  "file",
-						Usage: "Add a file logger",
-						Flags: append(defaultLoggingFlags, []cli.Flag{
-							cli.StringFlag{
-								Name:  "filename, f",
-								Usage: "Filename for the logger - this must be set.",
-							}, cli.BoolTFlag{
-								Name:  "rotate, r",
-								Usage: "Rotate logs",
-							}, cli.Int64Flag{
-								Name:  "max-size, s",
-								Usage: "Maximum size in bytes before rotation",
-							}, cli.BoolTFlag{
-								Name:  "daily, d",
-								Usage: "Rotate logs daily",
-							}, cli.IntFlag{
-								Name:  "max-days, D",
-								Usage: "Maximum number of daily logs to keep",
-							}, cli.BoolTFlag{
-								Name:  "compress, z",
-								Usage: "Compress rotated logs",
-							}, cli.IntFlag{
-								Name:  "compression-level, Z",
-								Usage: "Compression level to use",
-							},
-						}...),
-						Action: runAddFileLogger,
-					}, {
-						Name:  "conn",
-						Usage: "Add a net conn logger",
-						Flags: append(defaultLoggingFlags, []cli.Flag{
-							cli.BoolFlag{
-								Name:  "reconnect-on-message, R",
-								Usage: "Reconnect to host for every message",
-							}, cli.BoolFlag{
-								Name:  "reconnect, r",
-								Usage: "Reconnect to host when connection is dropped",
-							}, cli.StringFlag{
-								Name:  "protocol, P",
-								Usage: "Set protocol to use: tcp, unix, or udp (defaults to tcp)",
-							}, cli.StringFlag{
-								Name:  "address, a",
-								Usage: "Host address and port to connect to (defaults to :7020)",
-							},
-						}...),
-						Action: runAddConnLogger,
-					}, {
-						Name:  "smtp",
-						Usage: "Add an SMTP logger",
-						Flags: append(defaultLoggingFlags, []cli.Flag{
-							cli.StringFlag{
-								Name:  "username, u",
-								Usage: "Mail server username",
-							}, cli.StringFlag{
-								Name:  "password, P",
-								Usage: "Mail server password",
-							}, cli.StringFlag{
-								Name:  "host, H",
-								Usage: "Mail server host (defaults to: 127.0.0.1:25)",
-							}, cli.StringSliceFlag{
-								Name:  "send-to, s",
-								Usage: "Email address(es) to send to",
-							}, cli.StringFlag{
-								Name:  "subject, S",
-								Usage: "Subject header of sent emails",
-							},
-						}...),
-						Action: runAddSMTPLogger,
-					},
-				},
+	subCmdProcesses = cli.Command{
+		Name:   "processes",
+		Usage:  "Display running processes within the current process",
+		Action: runProcesses,
+		Flags: []cli.Flag{
+			cli.BoolFlag{
+				Name: "debug",
+			},
+			cli.BoolFlag{
+				Name:  "flat",
+				Usage: "Show processes as flat table rather than as tree",
+			},
+			cli.BoolFlag{
+				Name:  "no-system",
+				Usage: "Do not show system proceses",
+			},
+			cli.BoolFlag{
+				Name:  "stacktraces",
+				Usage: "Show stacktraces",
+			},
+			cli.BoolFlag{
+				Name:  "json",
+				Usage: "Output as json",
+			},
+			cli.StringFlag{
+				Name:  "cancel",
+				Usage: "Process PID to cancel. (Only available for non-system processes.)",
 			},
 		},
 	}
 )
 
-func runRemoveLogger(c *cli.Context) error {
-	setup("manager", c.Bool("debug"))
-	group := c.String("group")
-	if len(group) == 0 {
-		group = log.DEFAULT
-	}
-	name := c.Args().First()
-	ctx, cancel := installSignals()
-	defer cancel()
-
-	statusCode, msg := private.RemoveLogger(ctx, group, name)
-	switch statusCode {
-	case http.StatusInternalServerError:
-		return fail("InternalServerError", msg)
-	}
-
-	fmt.Fprintln(os.Stdout, msg)
-	return nil
-}
-
-func runAddSMTPLogger(c *cli.Context) error {
-	setup("manager", c.Bool("debug"))
-	vals := map[string]interface{}{}
-	mode := "smtp"
-	if c.IsSet("host") {
-		vals["host"] = c.String("host")
-	} else {
-		vals["host"] = "127.0.0.1:25"
-	}
-
-	if c.IsSet("username") {
-		vals["username"] = c.String("username")
-	}
-	if c.IsSet("password") {
-		vals["password"] = c.String("password")
-	}
-
-	if !c.IsSet("send-to") {
-		return fmt.Errorf("Some recipients must be provided")
-	}
-	vals["sendTos"] = c.StringSlice("send-to")
-
-	if c.IsSet("subject") {
-		vals["subject"] = c.String("subject")
-	} else {
-		vals["subject"] = "Diagnostic message from Gitea"
-	}
-
-	return commonAddLogger(c, mode, vals)
-}
-
-func runAddConnLogger(c *cli.Context) error {
-	setup("manager", c.Bool("debug"))
-	vals := map[string]interface{}{}
-	mode := "conn"
-	vals["net"] = "tcp"
-	if c.IsSet("protocol") {
-		switch c.String("protocol") {
-		case "udp":
-			vals["net"] = "udp"
-		case "unix":
-			vals["net"] = "unix"
-		}
-	}
-	if c.IsSet("address") {
-		vals["address"] = c.String("address")
-	} else {
-		vals["address"] = ":7020"
-	}
-	if c.IsSet("reconnect") {
-		vals["reconnect"] = c.Bool("reconnect")
-	}
-	if c.IsSet("reconnect-on-message") {
-		vals["reconnectOnMsg"] = c.Bool("reconnect-on-message")
-	}
-	return commonAddLogger(c, mode, vals)
-}
-
-func runAddFileLogger(c *cli.Context) error {
-	setup("manager", c.Bool("debug"))
-	vals := map[string]interface{}{}
-	mode := "file"
-	if c.IsSet("filename") {
-		vals["filename"] = c.String("filename")
-	} else {
-		return fmt.Errorf("filename must be set when creating a file logger")
-	}
-	if c.IsSet("rotate") {
-		vals["rotate"] = c.Bool("rotate")
-	}
-	if c.IsSet("max-size") {
-		vals["maxsize"] = c.Int64("max-size")
-	}
-	if c.IsSet("daily") {
-		vals["daily"] = c.Bool("daily")
-	}
-	if c.IsSet("max-days") {
-		vals["maxdays"] = c.Int("max-days")
-	}
-	if c.IsSet("compress") {
-		vals["compress"] = c.Bool("compress")
-	}
-	if c.IsSet("compression-level") {
-		vals["compressionLevel"] = c.Int("compression-level")
-	}
-	return commonAddLogger(c, mode, vals)
-}
-
-func runAddConsoleLogger(c *cli.Context) error {
-	setup("manager", c.Bool("debug"))
-	vals := map[string]interface{}{}
-	mode := "console"
-	if c.IsSet("stderr") && c.Bool("stderr") {
-		vals["stderr"] = c.Bool("stderr")
-	}
-	return commonAddLogger(c, mode, vals)
-}
-
-func commonAddLogger(c *cli.Context, mode string, vals map[string]interface{}) error {
-	if len(c.String("level")) > 0 {
-		vals["level"] = log.FromString(c.String("level")).String()
-	}
-	if len(c.String("stacktrace-level")) > 0 {
-		vals["stacktraceLevel"] = log.FromString(c.String("stacktrace-level")).String()
-	}
-	if len(c.String("expression")) > 0 {
-		vals["expression"] = c.String("expression")
-	}
-	if len(c.String("prefix")) > 0 {
-		vals["prefix"] = c.String("prefix")
-	}
-	if len(c.String("flags")) > 0 {
-		vals["flags"] = log.FlagsFromString(c.String("flags"))
-	}
-	if c.IsSet("color") {
-		vals["colorize"] = c.Bool("color")
-	}
-	group := "default"
-	if c.IsSet("group") {
-		group = c.String("group")
-	}
-	name := mode
-	if c.IsSet("name") {
-		name = c.String("name")
-	}
-	ctx, cancel := installSignals()
-	defer cancel()
-
-	statusCode, msg := private.AddLogger(ctx, group, name, mode, vals)
-	switch statusCode {
-	case http.StatusInternalServerError:
-		return fail("InternalServerError", msg)
-	}
-
-	fmt.Fprintln(os.Stdout, msg)
-	return nil
-}
-
 func runShutdown(c *cli.Context) error {
 	ctx, cancel := installSignals()
 	defer cancel()
@@ -433,47 +145,16 @@ func runFlushQueues(c *cli.Context) error {
 	return nil
 }
 
-func runPauseLogging(c *cli.Context) error {
+func runProcesses(c *cli.Context) error {
 	ctx, cancel := installSignals()
 	defer cancel()
 
 	setup("manager", c.Bool("debug"))
-	statusCode, msg := private.PauseLogging(ctx)
+	statusCode, msg := private.Processes(ctx, os.Stdout, c.Bool("flat"), c.Bool("no-system"), c.Bool("stacktraces"), c.Bool("json"), c.String("cancel"))
 	switch statusCode {
 	case http.StatusInternalServerError:
 		return fail("InternalServerError", msg)
 	}
 
-	fmt.Fprintln(os.Stdout, msg)
-	return nil
-}
-
-func runResumeLogging(c *cli.Context) error {
-	ctx, cancel := installSignals()
-	defer cancel()
-
-	setup("manager", c.Bool("debug"))
-	statusCode, msg := private.ResumeLogging(ctx)
-	switch statusCode {
-	case http.StatusInternalServerError:
-		return fail("InternalServerError", msg)
-	}
-
-	fmt.Fprintln(os.Stdout, msg)
-	return nil
-}
-
-func runReleaseReopenLogging(c *cli.Context) error {
-	ctx, cancel := installSignals()
-	defer cancel()
-
-	setup("manager", c.Bool("debug"))
-	statusCode, msg := private.ReleaseReopenLogging(ctx)
-	switch statusCode {
-	case http.StatusInternalServerError:
-		return fail("InternalServerError", msg)
-	}
-
-	fmt.Fprintln(os.Stdout, msg)
 	return nil
 }
diff --git a/cmd/manager_logging.go b/cmd/manager_logging.go
new file mode 100644
index 0000000000..eb311d2892
--- /dev/null
+++ b/cmd/manager_logging.go
@@ -0,0 +1,382 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cmd
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/private"
+	"github.com/urfave/cli"
+)
+
+var (
+	defaultLoggingFlags = []cli.Flag{
+		cli.StringFlag{
+			Name:  "group, g",
+			Usage: "Group to add logger to - will default to \"default\"",
+		}, cli.StringFlag{
+			Name:  "name, n",
+			Usage: "Name of the new logger - will default to mode",
+		}, cli.StringFlag{
+			Name:  "level, l",
+			Usage: "Logging level for the new logger",
+		}, cli.StringFlag{
+			Name:  "stacktrace-level, L",
+			Usage: "Stacktrace logging level",
+		}, cli.StringFlag{
+			Name:  "flags, F",
+			Usage: "Flags for the logger",
+		}, cli.StringFlag{
+			Name:  "expression, e",
+			Usage: "Matching expression for the logger",
+		}, cli.StringFlag{
+			Name:  "prefix, p",
+			Usage: "Prefix for the logger",
+		}, cli.BoolFlag{
+			Name:  "color",
+			Usage: "Use color in the logs",
+		}, cli.BoolFlag{
+			Name: "debug",
+		},
+	}
+
+	subcmdLogging = cli.Command{
+		Name:  "logging",
+		Usage: "Adjust logging commands",
+		Subcommands: []cli.Command{
+			{
+				Name:  "pause",
+				Usage: "Pause logging (Gitea will buffer logs up to a certain point and will drop them after that point)",
+				Flags: []cli.Flag{
+					cli.BoolFlag{
+						Name: "debug",
+					},
+				},
+				Action: runPauseLogging,
+			}, {
+				Name:  "resume",
+				Usage: "Resume logging",
+				Flags: []cli.Flag{
+					cli.BoolFlag{
+						Name: "debug",
+					},
+				},
+				Action: runResumeLogging,
+			}, {
+				Name:  "release-and-reopen",
+				Usage: "Cause Gitea to release and re-open files used for logging",
+				Flags: []cli.Flag{
+					cli.BoolFlag{
+						Name: "debug",
+					},
+				},
+				Action: runReleaseReopenLogging,
+			}, {
+				Name:      "remove",
+				Usage:     "Remove a logger",
+				ArgsUsage: "[name] Name of logger to remove",
+				Flags: []cli.Flag{
+					cli.BoolFlag{
+						Name: "debug",
+					}, cli.StringFlag{
+						Name:  "group, g",
+						Usage: "Group to add logger to - will default to \"default\"",
+					},
+				},
+				Action: runRemoveLogger,
+			}, {
+				Name:  "add",
+				Usage: "Add a logger",
+				Subcommands: []cli.Command{
+					{
+						Name:  "console",
+						Usage: "Add a console logger",
+						Flags: append(defaultLoggingFlags,
+							cli.BoolFlag{
+								Name:  "stderr",
+								Usage: "Output console logs to stderr - only relevant for console",
+							}),
+						Action: runAddConsoleLogger,
+					}, {
+						Name:  "file",
+						Usage: "Add a file logger",
+						Flags: append(defaultLoggingFlags, []cli.Flag{
+							cli.StringFlag{
+								Name:  "filename, f",
+								Usage: "Filename for the logger - this must be set.",
+							}, cli.BoolTFlag{
+								Name:  "rotate, r",
+								Usage: "Rotate logs",
+							}, cli.Int64Flag{
+								Name:  "max-size, s",
+								Usage: "Maximum size in bytes before rotation",
+							}, cli.BoolTFlag{
+								Name:  "daily, d",
+								Usage: "Rotate logs daily",
+							}, cli.IntFlag{
+								Name:  "max-days, D",
+								Usage: "Maximum number of daily logs to keep",
+							}, cli.BoolTFlag{
+								Name:  "compress, z",
+								Usage: "Compress rotated logs",
+							}, cli.IntFlag{
+								Name:  "compression-level, Z",
+								Usage: "Compression level to use",
+							},
+						}...),
+						Action: runAddFileLogger,
+					}, {
+						Name:  "conn",
+						Usage: "Add a net conn logger",
+						Flags: append(defaultLoggingFlags, []cli.Flag{
+							cli.BoolFlag{
+								Name:  "reconnect-on-message, R",
+								Usage: "Reconnect to host for every message",
+							}, cli.BoolFlag{
+								Name:  "reconnect, r",
+								Usage: "Reconnect to host when connection is dropped",
+							}, cli.StringFlag{
+								Name:  "protocol, P",
+								Usage: "Set protocol to use: tcp, unix, or udp (defaults to tcp)",
+							}, cli.StringFlag{
+								Name:  "address, a",
+								Usage: "Host address and port to connect to (defaults to :7020)",
+							},
+						}...),
+						Action: runAddConnLogger,
+					}, {
+						Name:  "smtp",
+						Usage: "Add an SMTP logger",
+						Flags: append(defaultLoggingFlags, []cli.Flag{
+							cli.StringFlag{
+								Name:  "username, u",
+								Usage: "Mail server username",
+							}, cli.StringFlag{
+								Name:  "password, P",
+								Usage: "Mail server password",
+							}, cli.StringFlag{
+								Name:  "host, H",
+								Usage: "Mail server host (defaults to: 127.0.0.1:25)",
+							}, cli.StringSliceFlag{
+								Name:  "send-to, s",
+								Usage: "Email address(es) to send to",
+							}, cli.StringFlag{
+								Name:  "subject, S",
+								Usage: "Subject header of sent emails",
+							},
+						}...),
+						Action: runAddSMTPLogger,
+					},
+				},
+			},
+		},
+	}
+)
+
+func runRemoveLogger(c *cli.Context) error {
+	setup("manager", c.Bool("debug"))
+	group := c.String("group")
+	if len(group) == 0 {
+		group = log.DEFAULT
+	}
+	name := c.Args().First()
+	ctx, cancel := installSignals()
+	defer cancel()
+
+	statusCode, msg := private.RemoveLogger(ctx, group, name)
+	switch statusCode {
+	case http.StatusInternalServerError:
+		return fail("InternalServerError", msg)
+	}
+
+	fmt.Fprintln(os.Stdout, msg)
+	return nil
+}
+
+func runAddSMTPLogger(c *cli.Context) error {
+	setup("manager", c.Bool("debug"))
+	vals := map[string]interface{}{}
+	mode := "smtp"
+	if c.IsSet("host") {
+		vals["host"] = c.String("host")
+	} else {
+		vals["host"] = "127.0.0.1:25"
+	}
+
+	if c.IsSet("username") {
+		vals["username"] = c.String("username")
+	}
+	if c.IsSet("password") {
+		vals["password"] = c.String("password")
+	}
+
+	if !c.IsSet("send-to") {
+		return fmt.Errorf("Some recipients must be provided")
+	}
+	vals["sendTos"] = c.StringSlice("send-to")
+
+	if c.IsSet("subject") {
+		vals["subject"] = c.String("subject")
+	} else {
+		vals["subject"] = "Diagnostic message from Gitea"
+	}
+
+	return commonAddLogger(c, mode, vals)
+}
+
+func runAddConnLogger(c *cli.Context) error {
+	setup("manager", c.Bool("debug"))
+	vals := map[string]interface{}{}
+	mode := "conn"
+	vals["net"] = "tcp"
+	if c.IsSet("protocol") {
+		switch c.String("protocol") {
+		case "udp":
+			vals["net"] = "udp"
+		case "unix":
+			vals["net"] = "unix"
+		}
+	}
+	if c.IsSet("address") {
+		vals["address"] = c.String("address")
+	} else {
+		vals["address"] = ":7020"
+	}
+	if c.IsSet("reconnect") {
+		vals["reconnect"] = c.Bool("reconnect")
+	}
+	if c.IsSet("reconnect-on-message") {
+		vals["reconnectOnMsg"] = c.Bool("reconnect-on-message")
+	}
+	return commonAddLogger(c, mode, vals)
+}
+
+func runAddFileLogger(c *cli.Context) error {
+	setup("manager", c.Bool("debug"))
+	vals := map[string]interface{}{}
+	mode := "file"
+	if c.IsSet("filename") {
+		vals["filename"] = c.String("filename")
+	} else {
+		return fmt.Errorf("filename must be set when creating a file logger")
+	}
+	if c.IsSet("rotate") {
+		vals["rotate"] = c.Bool("rotate")
+	}
+	if c.IsSet("max-size") {
+		vals["maxsize"] = c.Int64("max-size")
+	}
+	if c.IsSet("daily") {
+		vals["daily"] = c.Bool("daily")
+	}
+	if c.IsSet("max-days") {
+		vals["maxdays"] = c.Int("max-days")
+	}
+	if c.IsSet("compress") {
+		vals["compress"] = c.Bool("compress")
+	}
+	if c.IsSet("compression-level") {
+		vals["compressionLevel"] = c.Int("compression-level")
+	}
+	return commonAddLogger(c, mode, vals)
+}
+
+func runAddConsoleLogger(c *cli.Context) error {
+	setup("manager", c.Bool("debug"))
+	vals := map[string]interface{}{}
+	mode := "console"
+	if c.IsSet("stderr") && c.Bool("stderr") {
+		vals["stderr"] = c.Bool("stderr")
+	}
+	return commonAddLogger(c, mode, vals)
+}
+
+func commonAddLogger(c *cli.Context, mode string, vals map[string]interface{}) error {
+	if len(c.String("level")) > 0 {
+		vals["level"] = log.FromString(c.String("level")).String()
+	}
+	if len(c.String("stacktrace-level")) > 0 {
+		vals["stacktraceLevel"] = log.FromString(c.String("stacktrace-level")).String()
+	}
+	if len(c.String("expression")) > 0 {
+		vals["expression"] = c.String("expression")
+	}
+	if len(c.String("prefix")) > 0 {
+		vals["prefix"] = c.String("prefix")
+	}
+	if len(c.String("flags")) > 0 {
+		vals["flags"] = log.FlagsFromString(c.String("flags"))
+	}
+	if c.IsSet("color") {
+		vals["colorize"] = c.Bool("color")
+	}
+	group := "default"
+	if c.IsSet("group") {
+		group = c.String("group")
+	}
+	name := mode
+	if c.IsSet("name") {
+		name = c.String("name")
+	}
+	ctx, cancel := installSignals()
+	defer cancel()
+
+	statusCode, msg := private.AddLogger(ctx, group, name, mode, vals)
+	switch statusCode {
+	case http.StatusInternalServerError:
+		return fail("InternalServerError", msg)
+	}
+
+	fmt.Fprintln(os.Stdout, msg)
+	return nil
+}
+
+func runPauseLogging(c *cli.Context) error {
+	ctx, cancel := installSignals()
+	defer cancel()
+
+	setup("manager", c.Bool("debug"))
+	statusCode, msg := private.PauseLogging(ctx)
+	switch statusCode {
+	case http.StatusInternalServerError:
+		return fail("InternalServerError", msg)
+	}
+
+	fmt.Fprintln(os.Stdout, msg)
+	return nil
+}
+
+func runResumeLogging(c *cli.Context) error {
+	ctx, cancel := installSignals()
+	defer cancel()
+
+	setup("manager", c.Bool("debug"))
+	statusCode, msg := private.ResumeLogging(ctx)
+	switch statusCode {
+	case http.StatusInternalServerError:
+		return fail("InternalServerError", msg)
+	}
+
+	fmt.Fprintln(os.Stdout, msg)
+	return nil
+}
+
+func runReleaseReopenLogging(c *cli.Context) error {
+	ctx, cancel := installSignals()
+	defer cancel()
+
+	setup("manager", c.Bool("debug"))
+	statusCode, msg := private.ReleaseReopenLogging(ctx)
+	switch statusCode {
+	case http.StatusInternalServerError:
+		return fail("InternalServerError", msg)
+	}
+
+	fmt.Fprintln(os.Stdout, msg)
+	return nil
+}
diff --git a/cmd/web.go b/cmd/web.go
index 710c12775f..8c7c026172 100644
--- a/cmd/web.go
+++ b/cmd/web.go
@@ -16,6 +16,7 @@ import (
 
 	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/process"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/routers"
 	"code.gitea.io/gitea/routers/install"
@@ -59,6 +60,9 @@ and it takes care of all the other things for you`,
 }
 
 func runHTTPRedirector() {
+	_, _, finished := process.GetManager().AddTypedContext(graceful.GetManager().HammerContext(), "Web: HTTP Redirector", process.SystemProcessType, true)
+	defer finished()
+
 	source := fmt.Sprintf("%s:%s", setting.HTTPAddr, setting.PortToRedirect)
 	dest := strings.TrimSuffix(setting.AppURL, "/")
 	log.Info("Redirecting: %s to %s", source, dest)
@@ -141,8 +145,10 @@ func runWeb(ctx *cli.Context) error {
 
 	if setting.EnablePprof {
 		go func() {
+			_, _, finished := process.GetManager().AddTypedContext(context.Background(), "Web: PProf Server", process.SystemProcessType, true)
 			log.Info("Starting pprof server on localhost:6060")
 			log.Info("%v", http.ListenAndServe("localhost:6060", nil))
+			finished()
 		}()
 	}
 
@@ -204,6 +210,8 @@ func listen(m http.Handler, handleRedirector bool) error {
 	if setting.Protocol != setting.HTTPUnix && setting.Protocol != setting.FCGIUnix {
 		listenAddr = net.JoinHostPort(listenAddr, setting.HTTPPort)
 	}
+	_, _, finished := process.GetManager().AddTypedContext(graceful.GetManager().HammerContext(), "Web: Gitea Server", process.SystemProcessType, true)
+	defer finished()
 	log.Info("Listen: %v://%s%s", setting.Protocol, listenAddr, setting.AppSubURL)
 	// This can be useful for users, many users do wrong to their config and get strange behaviors behind a reverse-proxy.
 	// A user may fix the configuration mistake when he sees this log.
diff --git a/cmd/web_acme.go b/cmd/web_acme.go
index 459d4f0a76..7dbeb14a0e 100644
--- a/cmd/web_acme.go
+++ b/cmd/web_acme.go
@@ -15,6 +15,7 @@ import (
 
 	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/process"
 	"code.gitea.io/gitea/modules/setting"
 
 	"github.com/caddyserver/certmagic"
@@ -107,6 +108,9 @@ func runACME(listenAddr string, m http.Handler) error {
 
 	if enableHTTPChallenge {
 		go func() {
+			_, _, finished := process.GetManager().AddTypedContext(graceful.GetManager().HammerContext(), "Web: ACME HTTP challenge server", process.SystemProcessType, true)
+			defer finished()
+
 			log.Info("Running Let's Encrypt handler on %s", setting.HTTPAddr+":"+setting.PortToRedirect)
 			// all traffic coming into HTTP will be redirect to HTTPS automatically (LE HTTP-01 validation happens here)
 			err := runHTTP("tcp", setting.HTTPAddr+":"+setting.PortToRedirect, "Let's Encrypt HTTP Challenge", myACME.HTTPChallengeHandler(http.HandlerFunc(runLetsEncryptFallbackHandler)))