Before, Gitea shows the database table stats on the `admin dashboard` page. It has some problems: * `count(*)` is quite heavy. If tables have many records, this blocks loading the admin page blocks for a long time * Some users had even reported issues that they can't visit their admin page because this page causes blocking or `50x error (reverse proxy timeout)` * The `actions` stat is not useful. The table is simply too large. Does it really matter if it contains 1,000,000 rows or 9,999,999 rows? * The translation `admin.dashboard.statistic_info` is difficult to maintain. So, this PR uses a separate page to show the stats and removes the `actions` stat. ![image](https://github.com/go-gitea/gitea/assets/2114189/babf7c61-b93b-4a62-bfaa-22983636427e) ## :warning: BREAKING The `actions` Prometheus metrics collector has been removed for the reasons mentioned beforehand. Please do not rely on its output anymore.tags/v1.20.0-rc0
@@ -21,7 +21,7 @@ import ( | |||
type Statistic struct { | |||
Counter struct { | |||
User, Org, PublicKey, | |||
Repo, Watch, Star, Action, Access, | |||
Repo, Watch, Star, Access, | |||
Issue, IssueClosed, IssueOpen, | |||
Comment, Oauth, Follow, | |||
Mirror, Release, AuthSource, Webhook, | |||
@@ -55,7 +55,6 @@ func GetStatistic() (stats Statistic) { | |||
stats.Counter.Repo, _ = repo_model.CountRepositories(db.DefaultContext, repo_model.CountRepositoryOptions{}) | |||
stats.Counter.Watch, _ = e.Count(new(repo_model.Watch)) | |||
stats.Counter.Star, _ = e.Count(new(repo_model.Star)) | |||
stats.Counter.Action, _ = db.EstimateCount(db.DefaultContext, new(Action)) | |||
stats.Counter.Access, _ = e.Count(new(access_model.Access)) | |||
type IssueCount struct { | |||
@@ -83,7 +82,7 @@ func GetStatistic() (stats Statistic) { | |||
Find(&stats.Counter.IssueByRepository) | |||
} | |||
issueCounts := []IssueCount{} | |||
var issueCounts []IssueCount | |||
_ = e.Select("COUNT(*) AS count, is_closed").Table("issue").GroupBy("is_closed").Find(&issueCounts) | |||
for _, c := range issueCounts { |
@@ -9,7 +9,6 @@ import ( | |||
"xorm.io/builder" | |||
"xorm.io/xorm" | |||
"xorm.io/xorm/schemas" | |||
) | |||
// DefaultContext is the default context to run xorm queries in | |||
@@ -241,30 +240,6 @@ func TableName(bean interface{}) string { | |||
return x.TableName(bean) | |||
} | |||
// EstimateCount returns an estimate of total number of rows in table | |||
func EstimateCount(ctx context.Context, bean interface{}) (int64, error) { | |||
e := GetEngine(ctx) | |||
e.Context(ctx) | |||
var rows int64 | |||
var err error | |||
tablename := TableName(bean) | |||
switch x.Dialect().URI().DBType { | |||
case schemas.MYSQL: | |||
_, err = e.Context(ctx).SQL("SELECT table_rows FROM information_schema.tables WHERE tables.table_name = ? AND tables.table_schema = ?;", tablename, x.Dialect().URI().DBName).Get(&rows) | |||
case schemas.POSTGRES: | |||
// the table can live in multiple schemas of a postgres database | |||
// See https://wiki.postgresql.org/wiki/Count_estimate | |||
tablename = x.TableName(bean, true) | |||
_, err = e.Context(ctx).SQL("SELECT reltuples::bigint AS estimate FROM pg_class WHERE oid = ?::regclass;", tablename).Get(&rows) | |||
case schemas.MSSQL: | |||
_, err = e.Context(ctx).SQL("sp_spaceused ?;", tablename).Get(&rows) | |||
default: | |||
return e.Context(ctx).Count(tablename) | |||
} | |||
return rows, err | |||
} | |||
// InTransaction returns true if the engine is in a transaction otherwise return false | |||
func InTransaction(ctx context.Context) bool { | |||
_, ok := inTransaction(ctx) |
@@ -18,7 +18,6 @@ const namespace = "gitea_" | |||
// exposes gitea metrics for prometheus | |||
type Collector struct { | |||
Accesses *prometheus.Desc | |||
Actions *prometheus.Desc | |||
Attachments *prometheus.Desc | |||
BuildInfo *prometheus.Desc | |||
Comments *prometheus.Desc | |||
@@ -56,11 +55,6 @@ func NewCollector() Collector { | |||
"Number of Accesses", | |||
nil, nil, | |||
), | |||
Actions: prometheus.NewDesc( | |||
namespace+"actions", | |||
"Number of Actions", | |||
nil, nil, | |||
), | |||
Attachments: prometheus.NewDesc( | |||
namespace+"attachments", | |||
"Number of Attachments", | |||
@@ -207,7 +201,6 @@ func NewCollector() Collector { | |||
// Describe returns all possible prometheus.Desc | |||
func (c Collector) Describe(ch chan<- *prometheus.Desc) { | |||
ch <- c.Accesses | |||
ch <- c.Actions | |||
ch <- c.Attachments | |||
ch <- c.BuildInfo | |||
ch <- c.Comments | |||
@@ -246,11 +239,6 @@ func (c Collector) Collect(ch chan<- prometheus.Metric) { | |||
prometheus.GaugeValue, | |||
float64(stats.Counter.Access), | |||
) | |||
ch <- prometheus.MustNewConstMetric( | |||
c.Actions, | |||
prometheus.GaugeValue, | |||
float64(stats.Counter.Action), | |||
) | |||
ch <- prometheus.MustNewConstMetric( | |||
c.Attachments, | |||
prometheus.GaugeValue, |
@@ -2619,7 +2619,6 @@ dashboard.new_version_hint = Gitea %s is now available, you are running %s. Chec | |||
dashboard.statistic = Summary | |||
dashboard.operations = Maintenance Operations | |||
dashboard.system_status = System Status | |||
dashboard.statistic_info = The Gitea database holds <b>%d</b> users, <b>%d</b> organizations, <b>%d</b> public keys, <b>%d</b> repositories, <b>%d</b> watches, <b>%d</b> stars, ~<b>%d</b> actions, <b>%d</b> accesses, <b>%d</b> issues, <b>%d</b> comments, <b>%d</b> social accounts, <b>%d</b> follows, <b>%d</b> mirrors, <b>%d</b> releases, <b>%d</b> authentication sources, <b>%d</b> webhooks, <b>%d</b> milestones, <b>%d</b> labels, <b>%d</b> hook tasks, <b>%d</b> teams, <b>%d</b> update tasks, <b>%d</b> attachments. | |||
dashboard.operation_name = Operation Name | |||
dashboard.operation_switch = Switch | |||
dashboard.operation_run = Run | |||
@@ -3060,6 +3059,8 @@ config.xorm_log_sql = Log SQL | |||
config.get_setting_failed = Get setting %s failed | |||
config.set_setting_failed = Set setting %s failed | |||
monitor.stats = Stats | |||
monitor.cron = Cron Tasks | |||
monitor.name = Name | |||
monitor.schedule = Schedule |
@@ -8,11 +8,13 @@ import ( | |||
"fmt" | |||
"net/http" | |||
"runtime" | |||
"sort" | |||
"time" | |||
activities_model "code.gitea.io/gitea/models/activities" | |||
"code.gitea.io/gitea/modules/base" | |||
"code.gitea.io/gitea/modules/context" | |||
"code.gitea.io/gitea/modules/json" | |||
"code.gitea.io/gitea/modules/setting" | |||
"code.gitea.io/gitea/modules/updatechecker" | |||
"code.gitea.io/gitea/modules/web" | |||
@@ -26,6 +28,7 @@ const ( | |||
tplQueue base.TplName = "admin/queue" | |||
tplStacktrace base.TplName = "admin/stacktrace" | |||
tplQueueManage base.TplName = "admin/queue_manage" | |||
tplStats base.TplName = "admin/stats" | |||
) | |||
var sysStatus struct { | |||
@@ -111,7 +114,6 @@ func updateSystemStatus() { | |||
func Dashboard(ctx *context.Context) { | |||
ctx.Data["Title"] = ctx.Tr("admin.dashboard") | |||
ctx.Data["PageIsAdminDashboard"] = true | |||
ctx.Data["Stats"] = activities_model.GetStatistic() | |||
ctx.Data["NeedUpdate"] = updatechecker.GetNeedUpdate() | |||
ctx.Data["RemoteVersion"] = updatechecker.GetRemoteVersion() | |||
// FIXME: update periodically | |||
@@ -126,7 +128,6 @@ func DashboardPost(ctx *context.Context) { | |||
form := web.GetForm(ctx).(*forms.AdminDashboardForm) | |||
ctx.Data["Title"] = ctx.Tr("admin.dashboard") | |||
ctx.Data["PageIsAdminDashboard"] = true | |||
ctx.Data["Stats"] = activities_model.GetStatistic() | |||
updateSystemStatus() | |||
ctx.Data["SysStatus"] = sysStatus | |||
@@ -153,3 +154,30 @@ func CronTasks(ctx *context.Context) { | |||
ctx.Data["Entries"] = cron.ListTasks() | |||
ctx.HTML(http.StatusOK, tplCron) | |||
} | |||
func MonitorStats(ctx *context.Context) { | |||
ctx.Data["Title"] = ctx.Tr("admin.monitor.stats") | |||
ctx.Data["PageIsAdminMonitorStats"] = true | |||
bs, err := json.Marshal(activities_model.GetStatistic().Counter) | |||
if err != nil { | |||
ctx.ServerError("MonitorStats", err) | |||
return | |||
} | |||
statsCounter := map[string]any{} | |||
err = json.Unmarshal(bs, &statsCounter) | |||
if err != nil { | |||
ctx.ServerError("MonitorStats", err) | |||
return | |||
} | |||
statsKeys := make([]string, 0, len(statsCounter)) | |||
for k := range statsCounter { | |||
if statsCounter[k] == nil { | |||
continue | |||
} | |||
statsKeys = append(statsKeys, k) | |||
} | |||
sort.Strings(statsKeys) | |||
ctx.Data["StatsKeys"] = statsKeys | |||
ctx.Data["StatsCounter"] = statsCounter | |||
ctx.HTML(http.StatusOK, tplStats) | |||
} |
@@ -538,8 +538,8 @@ func registerRoutes(m *web.Route) { | |||
// ***** START: Admin ***** | |||
m.Group("/admin", func() { | |||
m.Get("", adminReq, admin.Dashboard) | |||
m.Post("", adminReq, web.Bind(forms.AdminDashboardForm{}), admin.DashboardPost) | |||
m.Get("", admin.Dashboard) | |||
m.Post("", web.Bind(forms.AdminDashboardForm{}), admin.DashboardPost) | |||
m.Group("/config", func() { | |||
m.Get("", admin.Config) | |||
@@ -548,6 +548,7 @@ func registerRoutes(m *web.Route) { | |||
}) | |||
m.Group("/monitor", func() { | |||
m.Get("/stats", admin.MonitorStats) | |||
m.Get("/cron", admin.CronTasks) | |||
m.Get("/stacktrace", admin.Stacktrace) | |||
m.Post("/stacktrace/cancel/{pid}", admin.StacktraceCancel) |
@@ -5,14 +5,6 @@ | |||
<p>{{(.locale.Tr "admin.dashboard.new_version_hint" .RemoteVersion AppVer) | Str2html}}</p> | |||
</div> | |||
{{end}} | |||
<h4 class="ui top attached header"> | |||
{{.locale.Tr "admin.dashboard.statistic"}} | |||
</h4> | |||
<div class="ui attached segment"> | |||
<p> | |||
{{.locale.Tr "admin.dashboard.statistic_info" .Stats.Counter.User .Stats.Counter.Org .Stats.Counter.PublicKey .Stats.Counter.Repo .Stats.Counter.Watch .Stats.Counter.Star .Stats.Counter.Action .Stats.Counter.Access .Stats.Counter.Issue .Stats.Counter.Comment .Stats.Counter.Oauth .Stats.Counter.Follow .Stats.Counter.Mirror .Stats.Counter.Release .Stats.Counter.AuthSource .Stats.Counter.Webhook .Stats.Counter.Milestone .Stats.Counter.Label .Stats.Counter.HookTask .Stats.Counter.Team .Stats.Counter.UpdateTask .Stats.Counter.Attachment | Str2html}} | |||
</p> | |||
</div> | |||
<h4 class="ui top attached header"> | |||
{{.locale.Tr "admin.dashboard.operations"}} | |||
</h4> |
@@ -53,6 +53,9 @@ | |||
<div class="item"> | |||
{{.locale.Tr "admin.monitor"}} | |||
<div class="menu"> | |||
<a class="{{if .PageIsAdminMonitorStats}}active {{end}}item" href="{{AppSubUrl}}/admin/monitor/stats"> | |||
{{.locale.Tr "admin.monitor.stats"}} | |||
</a> | |||
<a class="{{if .PageIsAdminMonitorCron}}active {{end}}item" href="{{AppSubUrl}}/admin/monitor/cron"> | |||
{{.locale.Tr "admin.monitor.cron"}} | |||
</a> |
@@ -0,0 +1,17 @@ | |||
{{template "admin/layout_head" (dict "ctxData" . "pageClass" "admin monitor")}} | |||
<div class="admin-setting-content"> | |||
<h4 class="ui top attached header"> | |||
{{.locale.Tr "admin.dashboard.statistic"}} | |||
</h4> | |||
<div class="ui attached table segment"> | |||
<table class="ui very basic striped table unstackable"> | |||
{{range $statsKey := .StatsKeys}} | |||
<tr> | |||
<td width="200">{{$statsKey}}</td> | |||
<td>{{index $.StatsCounter $statsKey}}</td> | |||
</tr> | |||
{{end}} | |||
</table> | |||
</div> | |||
</div> | |||
{{template "admin/layout_footer" .}} |