Improve queue & process & stacktrace (#24636)

Although some features are mixed together in this PR, this PR is not
that large, and these features are all related.

Actually there are more than 70 lines are for a toy "test queue", so
this PR is quite simple.

Major features:

1. Allow site admin to clear a queue (remove all items in a queue)
* Because there is no transaction, the "unique queue" could be corrupted
in rare cases, that's unfixable.
* eg: the item is in the "set" but not in the "list", so the item would
never be able to be pushed into the queue.
* Now site admin could simply clear the queue, then everything becomes
correct, the lost items could be re-pushed into queue by future
operations.
3. Split the "admin/monitor" to separate pages
4. Allow to download diagnosis report
* In history, there were many users reporting that Gitea queue gets
stuck, or Gitea's CPU is 100%
    * With diagnosis report, maintainers could know what happens clearly

The diagnosis report sample:
[gitea-diagnosis-20230510-192913.zip](https://github.com/go-gitea/gitea/files/11441346/gitea-diagnosis-20230510-192913.zip)
, use "go tool pprof profile.dat" to view the report.


Screenshots:


![image](https://github.com/go-gitea/gitea/assets/2114189/320659b4-2eda-4def-8dc0-5ea08d578063)


![image](https://github.com/go-gitea/gitea/assets/2114189/c5c46fae-9dc0-44ca-8cd3-57beedc5035e)


![image](https://github.com/go-gitea/gitea/assets/2114189/6168a811-42a1-4e64-a263-0617a6c8c4fe)

---------

Co-authored-by: Jason Song <i@wolfogre.com>
Co-authored-by: Giteabot <teabot@gitea.io>

8 files changed, 124 insertions, 129 deletions

diff --git a/templates/admin/cron.tmpl b/templates/admin/cron.tmpl
index a7f4405620..fe3c88aec1 100644
--- a/templates/admin/cron.tmpl
+++ b/templates/admin/cron.tmpl
@@ -1,35 +1,39 @@
-<h4 class="ui top attached header">
-	{{.locale.Tr "admin.monitor.cron"}}
-</h4>
-<div class="ui attached table segment">
-	<form method="post" action="{{AppSubUrl}}/admin">
-		<table class="ui very basic striped table unstackable">
-			<thead>
-				<tr>
-					<th></th>
-					<th>{{.locale.Tr "admin.monitor.name"}}</th>
-					<th>{{.locale.Tr "admin.monitor.schedule"}}</th>
-					<th>{{.locale.Tr "admin.monitor.next"}}</th>
-					<th>{{.locale.Tr "admin.monitor.previous"}}</th>
-					<th>{{.locale.Tr "admin.monitor.execute_times"}}</th>
-					<th>{{.locale.Tr "admin.monitor.last_execution_result"}}</th>
-				</tr>
-			</thead>
-			<tbody>
-				{{range .Entries}}
+{{template "admin/layout_head" (dict "ctxData" . "pageClass" "admin monitor")}}
+<div class="admin-setting-content">
+	<h4 class="ui top attached header">
+		{{.locale.Tr "admin.monitor.cron"}}
+	</h4>
+	<div class="ui attached table segment">
+		<form method="post" action="{{AppSubUrl}}/admin">
+			<table class="ui very basic striped table unstackable">
+				<thead>
 					<tr>
-						<td><button type="submit" class="ui green button" name="op" value="{{.Name}}" title="{{$.locale.Tr "admin.dashboard.operation_run"}}">{{svg "octicon-triangle-right"}}</button></td>
-						<td>{{$.locale.Tr (printf "admin.dashboard.%s" .Name)}}</td>
-						<td>{{.Spec}}</td>
-						<td>{{DateTime "full" .Next}}</td>
-						<td>{{if gt .Prev.Year 1}}{{DateTime "full" .Prev}}{{else}}-{{end}}</td>
-						<td>{{.ExecTimes}}</td>
-						<td {{if ne .Status ""}}data-tooltip-content="{{.FormatLastMessage $.locale}}"{{end}} >{{if eq .Status ""}}—{{else if eq .Status "finished"}}{{svg "octicon-check" 16}}{{else}}{{svg "octicon-x" 16}}{{end}}</td>
+						<th></th>
+						<th>{{.locale.Tr "admin.monitor.name"}}</th>
+						<th>{{.locale.Tr "admin.monitor.schedule"}}</th>
+						<th>{{.locale.Tr "admin.monitor.next"}}</th>
+						<th>{{.locale.Tr "admin.monitor.previous"}}</th>
+						<th>{{.locale.Tr "admin.monitor.execute_times"}}</th>
+						<th>{{.locale.Tr "admin.monitor.last_execution_result"}}</th>
 					</tr>
-				{{end}}
-			</tbody>
-		</table>
-		<input type="hidden" name="from" value="monitor">
-		{{.CsrfTokenHtml}}
-	</form>
+				</thead>
+				<tbody>
+					{{range .Entries}}
+						<tr>
+							<td><button type="submit" class="ui green button" name="op" value="{{.Name}}" title="{{$.locale.Tr "admin.dashboard.operation_run"}}">{{svg "octicon-triangle-right"}}</button></td>
+							<td>{{$.locale.Tr (printf "admin.dashboard.%s" .Name)}}</td>
+							<td>{{.Spec}}</td>
+							<td>{{DateTime "full" .Next}}</td>
+							<td>{{if gt .Prev.Year 1}}{{DateTime "full" .Prev}}{{else}}-{{end}}</td>
+							<td>{{.ExecTimes}}</td>
+							<td {{if ne .Status ""}}data-tooltip-content="{{.FormatLastMessage $.locale}}"{{end}} >{{if eq .Status ""}}—{{else if eq .Status "finished"}}{{svg "octicon-check" 16}}{{else}}{{svg "octicon-x" 16}}{{end}}</td>
+						</tr>
+					{{end}}
+				</tbody>
+			</table>
+			<input type="hidden" name="from" value="monitor">
+			{{.CsrfTokenHtml}}
+		</form>
+	</div>
 </div>
+{{template "admin/layout_footer" .}}
diff --git a/templates/admin/monitor.tmpl b/templates/admin/monitor.tmpl
deleted file mode 100644
index 6a13b7b552..0000000000
--- a/templates/admin/monitor.tmpl
+++ /dev/null
@@ -1,19 +0,0 @@
-{{template "admin/layout_head" (dict "ctxData" . "pageClass" "admin monitor")}}
-	<div class="admin-setting-content">
-		{{template "admin/cron" .}}
-		{{template "admin/queue" .}}
-		{{template "admin/process" .}}
-	</div>
-
-<div class="ui g-modal-confirm delete modal">
-	<div class="header">
-		{{.locale.Tr "admin.monitor.process.cancel"}}
-	</div>
-	<div class="content">
-		<p>{{$.locale.Tr "admin.monitor.process.cancel_notices" `<span class="name"></span>` | Safe}}</p>
-		<p>{{$.locale.Tr "admin.monitor.process.cancel_desc"}}</p>
-	</div>
-	{{template "base/modal_actions_confirm" .}}
-</div>
-
-{{template "admin/layout_footer" .}}
diff --git a/templates/admin/navbar.tmpl b/templates/admin/navbar.tmpl
index 51a93bacd7..eff5f1d83a 100644
--- a/templates/admin/navbar.tmpl
+++ b/templates/admin/navbar.tmpl
@@ -50,8 +50,19 @@
 		<a class="{{if .PageIsAdminNotices}}active {{end}}item" href="{{AppSubUrl}}/admin/notices">
 			{{.locale.Tr "admin.notices"}}
 		</a>
-		<a class="{{if .PageIsAdminMonitor}}active {{end}}item" href="{{AppSubUrl}}/admin/monitor">
+		<div class="item">
 			{{.locale.Tr "admin.monitor"}}
-		</a>
+			<div class="menu">
+				<a class="{{if .PageIsAdminMonitorCron}}active {{end}}item" href="{{AppSubUrl}}/admin/monitor/cron">
+					{{.locale.Tr "admin.monitor.cron"}}
+				</a>
+				<a class="{{if .PageIsAdminMonitorQueue}}active {{end}}item" href="{{AppSubUrl}}/admin/monitor/queue">
+					{{.locale.Tr "admin.monitor.queues"}}
+				</a>
+				<a class="{{if .PageIsAdminMonitorStacktrace}}active {{end}}item" href="{{AppSubUrl}}/admin/monitor/stacktrace">
+					{{.locale.Tr "admin.monitor.stacktrace"}}
+				</a>
+			</div>
+		</div>
 	</div>
 </div>
diff --git a/templates/admin/process-row.tmpl b/templates/admin/process-row.tmpl
deleted file mode 100644
index 8fd2d1af70..0000000000
--- a/templates/admin/process-row.tmpl
+++ /dev/null
@@ -1,23 +0,0 @@
-<div class="item">
-	<div class="gt-df gt-ac">
-		<div class="icon gt-ml-3 gt-mr-3">{{if eq .Process.Type "request"}}{{svg "octicon-globe" 16}}{{else if eq .Process.Type "system"}}{{svg "octicon-cpu" 16}}{{else}}{{svg "octicon-terminal" 16}}{{end}}</div>
-		<div class="content gt-f1">
-			<div class="header">{{.Process.Description}}</div>
-			<div class="description">{{TimeSince .Process.Start .root.locale}}</div>
-		</div>
-		<div>
-			{{if ne .Process.Type "system"}}
-				<a class="delete-button icon" href="" data-url="{{.root.Link}}/cancel/{{.Process.PID}}" data-id="{{.Process.PID}}" data-name="{{.Process.Description}}">{{svg "octicon-trash" 16 "text-red"}}</a>
-			{{end}}
-		</div>
-	</div>
-
-	{{$children := .Process.Children}}
-	{{if $children}}
-		<div class="divided list">
-			{{range $children}}
-				{{template "admin/process-row" dict "Process" . "root" $.root}}
-			{{end}}
-		</div>
-	{{end}}
-</div>
diff --git a/templates/admin/process.tmpl b/templates/admin/process.tmpl
deleted file mode 100644
index bd5bcfb3a2..0000000000
--- a/templates/admin/process.tmpl
+++ /dev/null
@@ -1,13 +0,0 @@
-<h4 class="ui top attached header">
-	{{.locale.Tr "admin.monitor.process"}}
-	<div class="ui right">
-		<a class="ui primary tiny button" href="{{AppSubUrl}}/admin/monitor/stacktrace">{{.locale.Tr "admin.monitor.stacktrace"}}</a>
-	</div>
-</h4>
-<div class="ui attached segment">
-	<div class="ui relaxed divided list">
-			{{range .Processes}}
-				{{template "admin/process-row" dict "Process" . "root" $}}
-			{{end}}
-	</div>
-</div>
diff --git a/templates/admin/queue.tmpl b/templates/admin/queue.tmpl
index 5bd507e69f..f58f25e4ae 100644
--- a/templates/admin/queue.tmpl
+++ b/templates/admin/queue.tmpl
@@ -1,29 +1,34 @@
-<h4 class="ui top attached header">
-	{{.locale.Tr "admin.monitor.queues"}}
-</h4>
-<div class="ui attached table segment">
-	<table class="ui very basic striped table unstackable">
-		<thead>
-		<tr>
-			<th>{{.locale.Tr "admin.monitor.queue.name"}}</th>
-			<th>{{.locale.Tr "admin.monitor.queue.type"}}</th>
-			<th>{{.locale.Tr "admin.monitor.queue.exemplar"}}</th>
-			<th>{{.locale.Tr "admin.monitor.queue.numberworkers"}}</th>
-			<th>{{.locale.Tr "admin.monitor.queue.numberinqueue"}}</th>
-			<th></th>
-		</tr>
-		</thead>
-		<tbody>
-		{{range $qid, $q := .Queues}}
-		<tr>
-			<td>{{$q.GetName}}</td>
-			<td>{{$q.GetType}}</td>
-			<td>{{$q.GetItemTypeName}}</td>
-			<td>{{$sum := $q.GetWorkerNumber}}{{if lt $sum 0}}-{{else}}{{$sum}}{{end}}</td>
-			<td>{{$sum = $q.GetQueueItemNumber}}{{if lt $sum 0}}-{{else}}{{$sum}}{{end}}</td>
-			<td><a href="{{$.Link}}/queue/{{$qid}}" class="button">{{if lt $sum 0}}{{$.locale.Tr "admin.monitor.queue.review"}}{{else}}{{$.locale.Tr "admin.monitor.queue.review_add"}}{{end}}</a>
-		</tr>
-		{{end}}
-		</tbody>
-	</table>
+{{template "admin/layout_head" (dict "ctxData" . "pageClass" "admin monitor")}}
+<div class="admin-setting-content">
+	<h4 class="ui top attached header">
+		{{.locale.Tr "admin.monitor.queues"}}
+	</h4>
+	<div class="ui attached table segment">
+		<table class="ui very basic striped table unstackable">
+			<thead>
+			<tr>
+				<th>{{.locale.Tr "admin.monitor.queue.name"}}</th>
+				<th>{{.locale.Tr "admin.monitor.queue.type"}}</th>
+				<th>{{.locale.Tr "admin.monitor.queue.exemplar"}}</th>
+				<th>{{.locale.Tr "admin.monitor.queue.numberworkers"}}</th>
+				<th>{{.locale.Tr "admin.monitor.queue.numberinqueue"}}</th>
+				<th></th>
+			</tr>
+			</thead>
+			<tbody>
+			{{range $qid, $q := .Queues}}
+			<tr>
+				<td>{{$q.GetName}}</td>
+				<td>{{$q.GetType}}</td>
+				<td>{{$q.GetItemTypeName}}</td>
+				<td>{{$sum := $q.GetWorkerNumber}}{{if lt $sum 0}}-{{else}}{{$sum}}{{end}}</td>
+				<td>{{$sum = $q.GetQueueItemNumber}}{{if lt $sum 0}}-{{else}}{{$sum}}{{end}}</td>
+				<td><a href="{{$.Link}}/{{$qid}}" class="button">{{if lt $sum 0}}{{$.locale.Tr "admin.monitor.queue.review"}}{{else}}{{$.locale.Tr "admin.monitor.queue.review_add"}}{{end}}</a>
+			</tr>
+			{{end}}
+			</tbody>
+		</table>
+	</div>
 </div>
+{{template "admin/layout_footer" .}}
+
diff --git a/templates/admin/queue_manage.tmpl b/templates/admin/queue_manage.tmpl
index 60d6b426b9..d456c2fbab 100644
--- a/templates/admin/queue_manage.tmpl
+++ b/templates/admin/queue_manage.tmpl
@@ -22,7 +22,18 @@
 						<td>{{.Queue.GetItemTypeName}}</td>
 						<td>{{$sum := .Queue.GetWorkerNumber}}{{if lt $sum 0}}-{{else}}{{$sum}}{{end}}</td>
 						<td>{{if lt $sum 0}}-{{else}}{{.Queue.GetWorkerMaxNumber}}{{end}}</td>
-						<td>{{$sum = .Queue.GetQueueItemNumber}}{{if lt $sum 0}}-{{else}}{{$sum}}{{end}}</td>
+						<td>
+							{{$sum = .Queue.GetQueueItemNumber}}
+							{{if lt $sum 0}}
+								-
+							{{else}}
+								{{$sum}}
+								<form action="{{$.Link}}/remove-all-items" method="post" class="gt-dib gt-ml-4">
+									{{$.CsrfTokenHtml}}
+									<button class="ui tiny basic red button">{{.locale.Tr "admin.monitor.queue.settings.remove_all_items"}}</button>
+								</form>
+							{{end}}
+						</td>
 					</tr>
 				</tbody>
 			</table>
diff --git a/templates/admin/stacktrace.tmpl b/templates/admin/stacktrace.tmpl
index 2ffb9982f2..46ad421587 100644
--- a/templates/admin/stacktrace.tmpl
+++ b/templates/admin/stacktrace.tmpl
@@ -1,19 +1,38 @@
 {{template "admin/layout_head" (dict "ctxData" . "pageClass" "admin monitor")}}
-	<div class="admin-setting-content">
-		<h4 class="ui top attached header">
-			{{.locale.Tr "admin.monitor.stacktrace"}}: {{.locale.Tr "admin.monitor.goroutines" .GoroutineCount}}
-			<div class="ui right">
-				<a class="ui primary tiny button" href="{{AppSubUrl}}/admin/monitor">{{.locale.Tr "admin.monitor"}}</a>
+<div class="admin-setting-content">
+
+	<div class="gt-df gt-ac">
+		<div class="gt-f1">
+			<div class="ui compact small menu">
+				<a class="{{if eq .ShowGoroutineList "process"}}active {{end}}item" href="{{.Link}}?show=process">{{.locale.Tr "admin.monitor.process"}}</a>
+				<a class="{{if eq .ShowGoroutineList "stacktrace"}}active {{end}}item" href="{{.Link}}?show=stacktrace">{{.locale.Tr "admin.monitor.stacktrace"}}</a>
 			</div>
-		</h4>
-		<div class="ui attached segment">
-			<div class="ui relaxed divided list">
-				{{range .ProcessStacks}}
-					{{template "admin/stacktrace-row" dict "Process" . "root" $}}
-				{{end}}
+		</div>
+		<form target="_blank" action="{{AppSubUrl}}/admin/monitor/diagnosis" class="ui form">
+			<div class="ui inline field">
+				<button class="ui primary small button">{{.locale.Tr "admin.monitor.download_diagnosis_report"}}</button>
+				<input name="seconds" size="3" maxlength="3" value="10"> {{.locale.Tr "tool.raw_seconds"}}
 			</div>
+		</form>
+	</div>
+
+	<div class="ui divider"></div>
+
+	<h4 class="ui top attached header">
+		{{printf "%d Goroutines" .GoroutineCount}}{{/* Goroutine is non-translatable*/}}
+		{{- if .ProcessCount -}}, {{.locale.Tr "admin.monitor.processes_count" .ProcessCount}}{{- end -}}
+	</h4>
+
+	{{if .ProcessStacks}}
+	<div class="ui attached segment">
+		<div class="ui relaxed divided list">
+			{{range .ProcessStacks}}
+				{{template "admin/stacktrace-row" dict "Process" . "root" $}}
+			{{end}}
 		</div>
 	</div>
+	{{end}}
+</div>
 
 <div class="ui g-modal-confirm delete modal">
 	<div class="header">