Refactor Cron and merge dashboard tasks (#10745)

* Refactor Cron and merge dashboard tasks

* Merge Cron and Dashboard tasks
* Make every cron task report a system notice on completion
* Refactor the creation of these tasks
* Ensure that execution counts of tasks is correct
* Allow cron tasks to be started from the cron page

* golangci-lint fixes

* Enforce that only one task with the same name can be registered

Signed-off-by: Andrew Thornton <art27@cantab.net>

* fix name check

Signed-off-by: Andrew Thornton <art27@cantab.net>

* as per @guillep2k

* as per @lafriks

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Add git.CommandContext variants

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: Lauris BH <lauris@nix.lv>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>

13 files changed, 632 insertions, 291 deletions

diff --git a/modules/auth/admin.go b/modules/auth/admin.go
index a772680680..af7197d2ae 100644
--- a/modules/auth/admin.go
+++ b/modules/auth/admin.go
@@ -51,7 +51,7 @@ func (f *AdminEditUserForm) Validate(ctx *macaron.Context, errs binding.Errors)
 
 // AdminDashboardForm form for admin dashboard operations
 type AdminDashboardForm struct {
-	Op int `binding:"required"`
+	Op string `binding:"required"`
 }
 
 // Validate validates form fields
diff --git a/modules/cron/cron.go b/modules/cron/cron.go
index 692642e4ce..ae309bd866 100644
--- a/modules/cron/cron.go
+++ b/modules/cron/cron.go
@@ -9,143 +9,86 @@ import (
 	"context"
 	"time"
 
-	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/graceful"
-	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/migrations"
-	repo_module "code.gitea.io/gitea/modules/repository"
-	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/sync"
-	mirror_service "code.gitea.io/gitea/services/mirror"
 
 	"github.com/gogs/cron"
 )
 
-const (
-	mirrorUpdate            = "mirror_update"
-	gitFsck                 = "git_fsck"
-	checkRepos              = "check_repos"
-	archiveCleanup          = "archive_cleanup"
-	syncExternalUsers       = "sync_external_users"
-	deletedBranchesCleanup  = "deleted_branches_cleanup"
-	updateMigrationPosterID = "update_migration_post_id"
-)
-
 var c = cron.New()
 
 // Prevent duplicate running tasks.
 var taskStatusTable = sync.NewStatusTable()
 
-// Func defines a cron function body
-type Func func()
-
-// WithUnique wrap a cron func with an unique running check
-func WithUnique(name string, body func(context.Context)) Func {
-	return func() {
-		if !taskStatusTable.StartIfNotRunning(name) {
-			return
-		}
-		defer taskStatusTable.Stop(name)
-		graceful.GetManager().RunWithShutdownContext(body)
-	}
-}
-
 // NewContext begins cron tasks
 // Each cron task is run within the shutdown context as a running server
 // AtShutdown the cron server is stopped
 func NewContext() {
-	var (
-		entry *cron.Entry
-		err   error
-	)
-	if setting.Cron.UpdateMirror.Enabled {
-		entry, err = c.AddFunc("Update mirrors", setting.Cron.UpdateMirror.Schedule, WithUnique(mirrorUpdate, mirror_service.Update))
-		if err != nil {
-			log.Fatal("Cron[Update mirrors]: %v", err)
-		}
-		if setting.Cron.UpdateMirror.RunAtStart {
-			entry.Prev = time.Now()
-			entry.ExecTimes++
-			go WithUnique(mirrorUpdate, mirror_service.Update)()
-		}
-	}
-	if setting.Cron.RepoHealthCheck.Enabled {
-		entry, err = c.AddFunc("Repository health check", setting.Cron.RepoHealthCheck.Schedule, WithUnique(gitFsck, func(ctx context.Context) {
-			if err := repo_module.GitFsck(ctx); err != nil {
-				log.Error("GitFsck: %s", err)
-			}
-		}))
-		if err != nil {
-			log.Fatal("Cron[Repository health check]: %v", err)
-		}
-		if setting.Cron.RepoHealthCheck.RunAtStart {
-			entry.Prev = time.Now()
-			entry.ExecTimes++
-			go WithUnique(gitFsck, func(ctx context.Context) {
-				if err := repo_module.GitFsck(ctx); err != nil {
-					log.Error("GitFsck: %s", err)
-				}
-			})()
-		}
-	}
-	if setting.Cron.CheckRepoStats.Enabled {
-		entry, err = c.AddFunc("Check repository statistics", setting.Cron.CheckRepoStats.Schedule, WithUnique(checkRepos, models.CheckRepoStats))
-		if err != nil {
-			log.Fatal("Cron[Check repository statistics]: %v", err)
-		}
-		if setting.Cron.CheckRepoStats.RunAtStart {
-			entry.Prev = time.Now()
-			entry.ExecTimes++
-			go WithUnique(checkRepos, models.CheckRepoStats)()
-		}
-	}
-	if setting.Cron.ArchiveCleanup.Enabled {
-		entry, err = c.AddFunc("Clean up old repository archives", setting.Cron.ArchiveCleanup.Schedule, WithUnique(archiveCleanup, models.DeleteOldRepositoryArchives))
-		if err != nil {
-			log.Fatal("Cron[Clean up old repository archives]: %v", err)
-		}
-		if setting.Cron.ArchiveCleanup.RunAtStart {
-			entry.Prev = time.Now()
-			entry.ExecTimes++
-			go WithUnique(archiveCleanup, models.DeleteOldRepositoryArchives)()
-		}
-	}
-	if setting.Cron.SyncExternalUsers.Enabled {
-		entry, err = c.AddFunc("Synchronize external users", setting.Cron.SyncExternalUsers.Schedule, WithUnique(syncExternalUsers, models.SyncExternalUsers))
-		if err != nil {
-			log.Fatal("Cron[Synchronize external users]: %v", err)
-		}
-		if setting.Cron.SyncExternalUsers.RunAtStart {
-			entry.Prev = time.Now()
-			entry.ExecTimes++
-			go WithUnique(syncExternalUsers, models.SyncExternalUsers)()
-		}
-	}
-	if setting.Cron.DeletedBranchesCleanup.Enabled {
-		entry, err = c.AddFunc("Remove old deleted branches", setting.Cron.DeletedBranchesCleanup.Schedule, WithUnique(deletedBranchesCleanup, models.RemoveOldDeletedBranches))
-		if err != nil {
-			log.Fatal("Cron[Remove old deleted branches]: %v", err)
-		}
-		if setting.Cron.DeletedBranchesCleanup.RunAtStart {
-			entry.Prev = time.Now()
-			entry.ExecTimes++
-			go WithUnique(deletedBranchesCleanup, models.RemoveOldDeletedBranches)()
-		}
-	}
+	initBasicTasks()
+	initExtendedTasks()
 
-	entry, err = c.AddFunc("Update migrated repositories' issues and comments' posterid", setting.Cron.UpdateMigrationPosterID.Schedule, WithUnique(updateMigrationPosterID, migrations.UpdateMigrationPosterID))
-	if err != nil {
-		log.Fatal("Cron[Update migrated repositories]: %v", err)
+	lock.Lock()
+	for _, task := range tasks {
+		if task.IsEnabled() && task.DoRunAtStart() {
+			go task.Run()
+		}
 	}
-	entry.Prev = time.Now()
-	entry.ExecTimes++
-	go WithUnique(updateMigrationPosterID, migrations.UpdateMigrationPosterID)()
 
 	c.Start()
-	graceful.GetManager().RunAtShutdown(context.Background(), c.Stop)
+	started = true
+	lock.Unlock()
+	graceful.GetManager().RunAtShutdown(context.Background(), func() {
+		c.Stop()
+		lock.Lock()
+		started = false
+		lock.Unlock()
+	})
+
+}
+
+// TaskTableRow represents a task row in the tasks table
+type TaskTableRow struct {
+	Name      string
+	Spec      string
+	Next      time.Time
+	Prev      time.Time
+	ExecTimes int64
 }
 
+// TaskTable represents a table of tasks
+type TaskTable []*TaskTableRow
+
 // ListTasks returns all running cron tasks.
-func ListTasks() []*cron.Entry {
-	return c.Entries()
+func ListTasks() TaskTable {
+	entries := c.Entries()
+	eMap := map[string]*cron.Entry{}
+	for _, e := range entries {
+		eMap[e.Description] = e
+	}
+	lock.Lock()
+	defer lock.Unlock()
+	tTable := make([]*TaskTableRow, 0, len(tasks))
+	for _, task := range tasks {
+		spec := "-"
+		var (
+			next time.Time
+			prev time.Time
+		)
+		if e, ok := eMap[task.Name]; ok {
+			spec = e.Spec
+			next = e.Next
+			prev = e.Prev
+		}
+		task.lock.Lock()
+		tTable = append(tTable, &TaskTableRow{
+			Name:      task.Name,
+			Spec:      spec,
+			Next:      next,
+			Prev:      prev,
+			ExecTimes: task.ExecTimes,
+		})
+		task.lock.Unlock()
+	}
+
+	return tTable
 }
diff --git a/modules/cron/setting.go b/modules/cron/setting.go
new file mode 100644
index 0000000000..dd93d03986
--- /dev/null
+++ b/modules/cron/setting.go
@@ -0,0 +1,72 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cron
+
+import (
+	"time"
+
+	"code.gitea.io/gitea/models"
+	"github.com/unknwon/i18n"
+)
+
+// Config represents a basic configuration interface that cron task
+type Config interface {
+	IsEnabled() bool
+	DoRunAtStart() bool
+	GetSchedule() string
+	FormatMessage(name, status string, doer *models.User, args ...interface{}) string
+}
+
+// BaseConfig represents the basic config for a Cron task
+type BaseConfig struct {
+	Enabled    bool
+	RunAtStart bool
+	Schedule   string
+}
+
+// OlderThanConfig represents a cron task with OlderThan setting
+type OlderThanConfig struct {
+	BaseConfig
+	OlderThan time.Duration
+}
+
+// UpdateExistingConfig represents a cron task with UpdateExisting setting
+type UpdateExistingConfig struct {
+	BaseConfig
+	UpdateExisting bool
+}
+
+// GetSchedule returns the schedule for the base config
+func (b *BaseConfig) GetSchedule() string {
+	return b.Schedule
+}
+
+// IsEnabled returns the enabled status for the config
+func (b *BaseConfig) IsEnabled() bool {
+	return b.Enabled
+}
+
+// DoRunAtStart returns whether the task should be run at the start
+func (b *BaseConfig) DoRunAtStart() bool {
+	return b.RunAtStart
+}
+
+// FormatMessage returns a message for the task
+func (b *BaseConfig) FormatMessage(name, status string, doer *models.User, args ...interface{}) string {
+	realArgs := make([]interface{}, 0, len(args)+2)
+	realArgs = append(realArgs, i18n.Tr("en-US", "admin.dashboard."+name))
+	if doer == nil {
+		realArgs = append(realArgs, "(Cron)")
+	} else {
+		realArgs = append(realArgs, doer.Name)
+	}
+	if len(args) > 0 {
+		realArgs = append(realArgs, args...)
+	}
+	if doer == nil || (doer.ID == -1 && doer.Name == "(Cron)") {
+		return i18n.Tr("en-US", "admin.dashboard.cron."+status, realArgs...)
+	}
+	return i18n.Tr("en-US", "admin.dashboard.task."+status, realArgs...)
+}
diff --git a/modules/cron/tasks.go b/modules/cron/tasks.go
new file mode 100644
index 0000000000..a97326bd0f
--- /dev/null
+++ b/modules/cron/tasks.go
@@ -0,0 +1,166 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cron
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"sync"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/graceful"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/process"
+	"code.gitea.io/gitea/modules/setting"
+)
+
+var lock = sync.Mutex{}
+var started = false
+var tasks = []*Task{}
+var tasksMap = map[string]*Task{}
+
+// Task represents a Cron task
+type Task struct {
+	lock      sync.Mutex
+	Name      string
+	config    Config
+	fun       func(context.Context, *models.User, Config) error
+	ExecTimes int64
+}
+
+// DoRunAtStart returns if this task should run at the start
+func (t *Task) DoRunAtStart() bool {
+	return t.config.DoRunAtStart()
+}
+
+// IsEnabled returns if this task is enabled as cron task
+func (t *Task) IsEnabled() bool {
+	return t.config.IsEnabled()
+}
+
+// GetConfig will return a copy of the task's config
+func (t *Task) GetConfig() Config {
+	if reflect.TypeOf(t.config).Kind() == reflect.Ptr {
+		// Pointer:
+		return reflect.New(reflect.ValueOf(t.config).Elem().Type()).Interface().(Config)
+	}
+	// Not pointer:
+	return reflect.New(reflect.TypeOf(t.config)).Elem().Interface().(Config)
+}
+
+// Run will run the task incrementing the cron counter with no user defined
+func (t *Task) Run() {
+	t.RunWithUser(&models.User{
+		ID:        -1,
+		Name:      "(Cron)",
+		LowerName: "(cron)",
+	}, t.config)
+}
+
+// RunWithUser will run the task incrementing the cron counter at the time with User
+func (t *Task) RunWithUser(doer *models.User, config Config) {
+	if !taskStatusTable.StartIfNotRunning(t.Name) {
+		return
+	}
+	t.lock.Lock()
+	if config == nil {
+		config = t.config
+	}
+	t.ExecTimes++
+	t.lock.Unlock()
+	defer func() {
+		taskStatusTable.Stop(t.Name)
+		if err := recover(); err != nil {
+			// Recover a panic within the
+			combinedErr := fmt.Errorf("%s\n%s", err, log.Stack(2))
+			log.Error("PANIC whilst running task: %s Value: %v", t.Name, combinedErr)
+		}
+	}()
+	graceful.GetManager().RunWithShutdownContext(func(baseCtx context.Context) {
+		ctx, cancel := context.WithCancel(baseCtx)
+		defer cancel()
+		pm := process.GetManager()
+		pid := pm.Add(config.FormatMessage(t.Name, "process", doer), cancel)
+		defer pm.Remove(pid)
+		if err := t.fun(ctx, doer, config); err != nil {
+			if models.IsErrCancelled(err) {
+				message := err.(models.ErrCancelled).Message
+				if err := models.CreateNotice(models.NoticeTask, config.FormatMessage(t.Name, "aborted", doer, message)); err != nil {
+					log.Error("CreateNotice: %v", err)
+				}
+				return
+			}
+			if err := models.CreateNotice(models.NoticeTask, config.FormatMessage(t.Name, "error", doer, err)); err != nil {
+				log.Error("CreateNotice: %v", err)
+			}
+			return
+		}
+		if err := models.CreateNotice(models.NoticeTask, config.FormatMessage(t.Name, "finished", doer)); err != nil {
+			log.Error("CreateNotice: %v", err)
+		}
+	})
+}
+
+// GetTask gets the named task
+func GetTask(name string) *Task {
+	lock.Lock()
+	defer lock.Unlock()
+	log.Info("Getting %s in %v", name, tasksMap[name])
+
+	return tasksMap[name]
+}
+
+// RegisterTask allows a task to be registered with the cron service
+func RegisterTask(name string, config Config, fun func(context.Context, *models.User, Config) error) error {
+	log.Debug("Registering task: %s", name)
+	_, err := setting.GetCronSettings(name, config)
+	if err != nil {
+		log.Error("Unable to register cron task with name: %s Error: %v", name, err)
+		return err
+	}
+
+	task := &Task{
+		Name:   name,
+		config: config,
+		fun:    fun,
+	}
+	lock.Lock()
+	locked := true
+	defer func() {
+		if locked {
+			lock.Unlock()
+		}
+	}()
+	if _, has := tasksMap[task.Name]; has {
+		log.Error("A task with this name: %s has already been registered", name)
+		return fmt.Errorf("duplicate task with name: %s", task.Name)
+	}
+
+	if config.IsEnabled() {
+		// We cannot use the entry return as there is no way to lock it
+		if _, err = c.AddJob(name, config.GetSchedule(), task); err != nil {
+			log.Error("Unable to register cron task with name: %s Error: %v", name, err)
+			return err
+		}
+	}
+
+	tasks = append(tasks, task)
+	tasksMap[task.Name] = task
+	if started && config.IsEnabled() && config.DoRunAtStart() {
+		lock.Unlock()
+		locked = false
+		task.Run()
+	}
+
+	return nil
+}
+
+// RegisterTaskFatal will register a task but if there is an error log.Fatal
+func RegisterTaskFatal(name string, config Config, fun func(context.Context, *models.User, Config) error) {
+	if err := RegisterTask(name, config, fun); err != nil {
+		log.Fatal("Unable to register cron task %s Error: %v", name, err)
+	}
+}
diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go
new file mode 100644
index 0000000000..438c4a5004
--- /dev/null
+++ b/modules/cron/tasks_basic.go
@@ -0,0 +1,118 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cron
+
+import (
+	"context"
+	"time"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/migrations"
+	repository_service "code.gitea.io/gitea/modules/repository"
+	mirror_service "code.gitea.io/gitea/services/mirror"
+)
+
+func registerUpdateMirrorTask() {
+	RegisterTaskFatal("update_mirrors", &BaseConfig{
+		Enabled:    true,
+		RunAtStart: false,
+		Schedule:   "@every 10m",
+	}, func(ctx context.Context, _ *models.User, _ Config) error {
+		return mirror_service.Update(ctx)
+	})
+}
+
+func registerRepoHealthCheck() {
+	type RepoHealthCheckConfig struct {
+		BaseConfig
+		Timeout time.Duration
+		Args    []string `delim:" "`
+	}
+	RegisterTaskFatal("repo_health_check", &RepoHealthCheckConfig{
+		BaseConfig: BaseConfig{
+			Enabled:    true,
+			RunAtStart: false,
+			Schedule:   "@every 24h",
+		},
+		Timeout: 60 * time.Second,
+		Args:    []string{},
+	}, func(ctx context.Context, _ *models.User, config Config) error {
+		rhcConfig := config.(*RepoHealthCheckConfig)
+		return repository_service.GitFsck(ctx, rhcConfig.Timeout, rhcConfig.Args)
+	})
+}
+
+func registerCheckRepoStats() {
+	RegisterTaskFatal("check_repo_stats", &BaseConfig{
+		Enabled:    true,
+		RunAtStart: true,
+		Schedule:   "@every 24h",
+	}, func(ctx context.Context, _ *models.User, _ Config) error {
+		return models.CheckRepoStats(ctx)
+	})
+}
+
+func registerArchiveCleanup() {
+	RegisterTaskFatal("archive_cleanup", &OlderThanConfig{
+		BaseConfig: BaseConfig{
+			Enabled:    true,
+			RunAtStart: true,
+			Schedule:   "@every 24h",
+		},
+		OlderThan: 24 * time.Hour,
+	}, func(ctx context.Context, _ *models.User, config Config) error {
+		acConfig := config.(*OlderThanConfig)
+		return models.DeleteOldRepositoryArchives(ctx, acConfig.OlderThan)
+	})
+}
+
+func registerSyncExternalUsers() {
+	RegisterTaskFatal("sync_external_users", &UpdateExistingConfig{
+		BaseConfig: BaseConfig{
+			Enabled:    true,
+			RunAtStart: false,
+			Schedule:   "@every 24h",
+		},
+		UpdateExisting: true,
+	}, func(ctx context.Context, _ *models.User, config Config) error {
+		realConfig := config.(*UpdateExistingConfig)
+		return models.SyncExternalUsers(ctx, realConfig.UpdateExisting)
+	})
+}
+
+func registerDeletedBranchesCleanup() {
+	RegisterTaskFatal("deleted_branches_cleanup", &OlderThanConfig{
+		BaseConfig: BaseConfig{
+			Enabled:    true,
+			RunAtStart: true,
+			Schedule:   "@every 24h",
+		},
+		OlderThan: 24 * time.Hour,
+	}, func(ctx context.Context, _ *models.User, config Config) error {
+		realConfig := config.(*OlderThanConfig)
+		models.RemoveOldDeletedBranches(ctx, realConfig.OlderThan)
+		return nil
+	})
+}
+
+func registerUpdateMigrationPosterID() {
+	RegisterTaskFatal("update_migration_poster_id", &BaseConfig{
+		Enabled:    true,
+		RunAtStart: true,
+		Schedule:   "@every 24h",
+	}, func(ctx context.Context, _ *models.User, _ Config) error {
+		return migrations.UpdateMigrationPosterID(ctx)
+	})
+}
+
+func initBasicTasks() {
+	registerUpdateMirrorTask()
+	registerRepoHealthCheck()
+	registerCheckRepoStats()
+	registerArchiveCleanup()
+	registerSyncExternalUsers()
+	registerDeletedBranchesCleanup()
+	registerUpdateMigrationPosterID()
+}
diff --git a/modules/cron/tasks_extended.go b/modules/cron/tasks_extended.go
new file mode 100644
index 0000000000..fa2d6e0c38
--- /dev/null
+++ b/modules/cron/tasks_extended.go
@@ -0,0 +1,119 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cron
+
+import (
+	"context"
+	"time"
+
+	"code.gitea.io/gitea/models"
+	repo_module "code.gitea.io/gitea/modules/repository"
+	"code.gitea.io/gitea/modules/setting"
+)
+
+func registerDeleteInactiveUsers() {
+	RegisterTaskFatal("delete_inactive_accounts", &OlderThanConfig{
+		BaseConfig: BaseConfig{
+			Enabled:    false,
+			RunAtStart: false,
+			Schedule:   "@annually",
+		},
+		OlderThan: 0 * time.Second,
+	}, func(ctx context.Context, _ *models.User, config Config) error {
+		olderThanConfig := config.(*OlderThanConfig)
+		return models.DeleteInactiveUsers(ctx, olderThanConfig.OlderThan)
+	})
+}
+
+func registerDeleteRepositoryArchives() {
+	RegisterTaskFatal("delete_repo_archives", &BaseConfig{
+		Enabled:    false,
+		RunAtStart: false,
+		Schedule:   "@annually",
+	}, func(ctx context.Context, _ *models.User, _ Config) error {
+		return models.DeleteRepositoryArchives(ctx)
+	})
+}
+
+func registerGarbageCollectRepositories() {
+	type RepoHealthCheckConfig struct {
+		BaseConfig
+		Timeout time.Duration
+		Args    []string `delim:" "`
+	}
+	RegisterTaskFatal("git_gc_repos", &RepoHealthCheckConfig{
+		BaseConfig: BaseConfig{
+			Enabled:    false,
+			RunAtStart: false,
+			Schedule:   "@every 72h",
+		},
+		Timeout: time.Duration(setting.Git.Timeout.GC) * time.Second,
+		Args:    setting.Git.GCArgs,
+	}, func(ctx context.Context, _ *models.User, config Config) error {
+		rhcConfig := config.(*RepoHealthCheckConfig)
+		return repo_module.GitGcRepos(ctx, rhcConfig.Timeout, rhcConfig.Args...)
+	})
+}
+
+func registerRewriteAllPublicKeys() {
+	RegisterTaskFatal("resync_all_sshkeys", &BaseConfig{
+		Enabled:    false,
+		RunAtStart: false,
+		Schedule:   "@every 72h",
+	}, func(_ context.Context, _ *models.User, _ Config) error {
+		return models.RewriteAllPublicKeys()
+	})
+}
+
+func registerRepositoryUpdateHook() {
+	RegisterTaskFatal("resync_all_hooks", &BaseConfig{
+		Enabled:    false,
+		RunAtStart: false,
+		Schedule:   "@every 72h",
+	}, func(ctx context.Context, _ *models.User, _ Config) error {
+		return repo_module.SyncRepositoryHooks(ctx)
+	})
+}
+
+func registerReinitMissingRepositories() {
+	RegisterTaskFatal("reinit_missing_repos", &BaseConfig{
+		Enabled:    false,
+		RunAtStart: false,
+		Schedule:   "@every 72h",
+	}, func(ctx context.Context, _ *models.User, _ Config) error {
+		return repo_module.ReinitMissingRepositories(ctx)
+	})
+}
+
+func registerDeleteMissingRepositories() {
+	RegisterTaskFatal("delete_missing_repos", &BaseConfig{
+		Enabled:    false,
+		RunAtStart: false,
+		Schedule:   "@every 72h",
+	}, func(ctx context.Context, user *models.User, _ Config) error {
+		return repo_module.DeleteMissingRepositories(ctx, user)
+	})
+}
+
+func registerRemoveRandomAvatars() {
+	RegisterTaskFatal("delete_generated_repository_avatars", &BaseConfig{
+		Enabled:    false,
+		RunAtStart: false,
+		Schedule:   "@every 72h",
+	}, func(ctx context.Context, _ *models.User, _ Config) error {
+		return models.RemoveRandomAvatars(ctx)
+	})
+}
+
+func initExtendedTasks() {
+	registerDeleteInactiveUsers()
+	registerDeleteRepositoryArchives()
+	registerGarbageCollectRepositories()
+	registerRewriteAllPublicKeys()
+	registerRepositoryUpdateHook()
+	registerReinitMissingRepositories()
+	registerDeleteMissingRepositories()
+	registerRemoveRandomAvatars()
+}
diff --git a/modules/git/command.go b/modules/git/command.go
index 6c931790c0..14fab4ef3c 100644
--- a/modules/git/command.go
+++ b/modules/git/command.go
@@ -45,22 +45,32 @@ func (c *Command) String() string {
 
 // NewCommand creates and returns a new Git Command based on given command and arguments.
 func NewCommand(args ...string) *Command {
+	return NewCommandContext(DefaultContext, args...)
+}
+
+// NewCommandContext creates and returns a new Git Command based on given command and arguments.
+func NewCommandContext(ctx context.Context, args ...string) *Command {
 	// Make an explicit copy of GlobalCommandArgs, otherwise append might overwrite it
 	cargs := make([]string, len(GlobalCommandArgs))
 	copy(cargs, GlobalCommandArgs)
 	return &Command{
 		name:          GitExecutable,
 		args:          append(cargs, args...),
-		parentContext: DefaultContext,
+		parentContext: ctx,
 	}
 }
 
 // NewCommandNoGlobals creates and returns a new Git Command based on given command and arguments only with the specify args and don't care global command args
 func NewCommandNoGlobals(args ...string) *Command {
+	return NewCommandContextNoGlobals(DefaultContext, args...)
+}
+
+// NewCommandContextNoGlobals creates and returns a new Git Command based on given command and arguments only with the specify args and don't care global command args
+func NewCommandContextNoGlobals(ctx context.Context, args ...string) *Command {
 	return &Command{
 		name:          GitExecutable,
 		args:          args,
-		parentContext: DefaultContext,
+		parentContext: ctx,
 	}
 }
 
diff --git a/modules/git/git.go b/modules/git/git.go
index d5caaa0912..7f718511f7 100644
--- a/modules/git/git.go
+++ b/modules/git/git.go
@@ -150,11 +150,11 @@ func Init(ctx context.Context) error {
 }
 
 // Fsck verifies the connectivity and validity of the objects in the database
-func Fsck(repoPath string, timeout time.Duration, args ...string) error {
+func Fsck(ctx context.Context, repoPath string, timeout time.Duration, args ...string) error {
 	// Make sure timeout makes sense.
 	if timeout <= 0 {
 		timeout = -1
 	}
-	_, err := NewCommand("fsck").AddArguments(args...).RunInDirTimeout(timeout, repoPath)
+	_, err := NewCommandContext(ctx, "fsck").AddArguments(args...).RunInDirTimeout(timeout, repoPath)
 	return err
 }
diff --git a/modules/migrations/update.go b/modules/migrations/update.go
index 3d0962657c..e7a57ceca8 100644
--- a/modules/migrations/update.go
+++ b/modules/migrations/update.go
@@ -13,17 +13,19 @@ import (
 )
 
 // UpdateMigrationPosterID updates all migrated repositories' issues and comments posterID
-func UpdateMigrationPosterID(ctx context.Context) {
+func UpdateMigrationPosterID(ctx context.Context) error {
 	for _, gitService := range structs.SupportedFullGitService {
 		select {
 		case <-ctx.Done():
-			log.Warn("UpdateMigrationPosterID aborted due to shutdown before %s", gitService.Name())
+			log.Warn("UpdateMigrationPosterID aborted before %s", gitService.Name())
+			return models.ErrCancelledf("during UpdateMigrationPosterID before %s", gitService.Name())
 		default:
 		}
 		if err := updateMigrationPosterIDByGitService(ctx, gitService); err != nil {
 			log.Error("updateMigrationPosterIDByGitService failed: %v", err)
 		}
 	}
+	return nil
 }
 
 func updateMigrationPosterIDByGitService(ctx context.Context, tp structs.GitServiceType) error {
@@ -37,7 +39,7 @@ func updateMigrationPosterIDByGitService(ctx context.Context, tp structs.GitServ
 	for {
 		select {
 		case <-ctx.Done():
-			log.Warn("UpdateMigrationPosterIDByGitService(%s) aborted due to shutdown", tp.Name())
+			log.Warn("UpdateMigrationPosterIDByGitService(%s) cancelled", tp.Name())
 			return nil
 		default:
 		}
@@ -54,7 +56,7 @@ func updateMigrationPosterIDByGitService(ctx context.Context, tp structs.GitServ
 		for _, user := range users {
 			select {
 			case <-ctx.Done():
-				log.Warn("UpdateMigrationPosterIDByGitService(%s) aborted due to shutdown", tp.Name())
+				log.Warn("UpdateMigrationPosterIDByGitService(%s) cancelled", tp.Name())
 				return nil
 			default:
 			}
diff --git a/modules/repository/check.go b/modules/repository/check.go
index fcaf76308f..90186d6a29 100644
--- a/modules/repository/check.go
+++ b/modules/repository/check.go
@@ -7,19 +7,19 @@ package repository
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/setting"
 
 	"github.com/unknwon/com"
 	"xorm.io/builder"
 )
 
 // GitFsck calls 'git fsck' to check repository health.
-func GitFsck(ctx context.Context) error {
+func GitFsck(ctx context.Context, timeout time.Duration, args []string) error {
 	log.Trace("Doing: GitFsck")
 
 	if err := models.Iterate(
@@ -27,24 +27,24 @@ func GitFsck(ctx context.Context) error {
 		new(models.Repository),
 		builder.Expr("id>0 AND is_fsck_enabled=?", true),
 		func(idx int, bean interface{}) error {
+			repo := bean.(*models.Repository)
 			select {
 			case <-ctx.Done():
-				return fmt.Errorf("Aborted due to shutdown")
+				return models.ErrCancelledf("before fsck of %s", repo.FullName())
 			default:
 			}
-			repo := bean.(*models.Repository)
+			log.Trace("Running health check on repository %v", repo)
 			repoPath := repo.RepoPath()
-			log.Trace("Running health check on repository %s", repoPath)
-			if err := git.Fsck(repoPath, setting.Cron.RepoHealthCheck.Timeout, setting.Cron.RepoHealthCheck.Args...); err != nil {
-				desc := fmt.Sprintf("Failed to health check repository (%s): %v", repoPath, err)
-				log.Warn(desc)
-				if err = models.CreateRepositoryNotice(desc); err != nil {
+			if err := git.Fsck(ctx, repoPath, timeout, args...); err != nil {
+				log.Warn("Failed to health check repository (%v): %v", repo, err)
+				if err = models.CreateRepositoryNotice("Failed to health check repository (%s): %v", repo.FullName(), err); err != nil {
 					log.Error("CreateRepositoryNotice: %v", err)
 				}
 			}
 			return nil
 		},
 	); err != nil {
+		log.Trace("Error: GitFsck: %v", err)
 		return err
 	}
 
@@ -53,32 +53,43 @@ func GitFsck(ctx context.Context) error {
 }
 
 // GitGcRepos calls 'git gc' to remove unnecessary files and optimize the local repository
-func GitGcRepos(ctx context.Context) error {
+func GitGcRepos(ctx context.Context, timeout time.Duration, args ...string) error {
 	log.Trace("Doing: GitGcRepos")
-	args := append([]string{"gc"}, setting.Git.GCArgs...)
+	args = append([]string{"gc"}, args...)
 
 	if err := models.Iterate(
 		models.DefaultDBContext(),
 		new(models.Repository),
 		builder.Gt{"id": 0},
 		func(idx int, bean interface{}) error {
+			repo := bean.(*models.Repository)
 			select {
 			case <-ctx.Done():
-				return fmt.Errorf("Aborted due to shutdown")
+				return models.ErrCancelledf("before GC of %s", repo.FullName())
 			default:
 			}
-
-			repo := bean.(*models.Repository)
-			if err := repo.GetOwner(); err != nil {
-				return err
+			log.Trace("Running git gc on %v", repo)
+			command := git.NewCommandContext(ctx, args...).
+				SetDescription(fmt.Sprintf("Repository Garbage Collection: %s", repo.FullName()))
+			var stdout string
+			var err error
+			if timeout > 0 {
+				var stdoutBytes []byte
+				stdoutBytes, err = command.RunInDirTimeout(
+					timeout,
+					repo.RepoPath())
+				stdout = string(stdoutBytes)
+			} else {
+				stdout, err = command.RunInDir(repo.RepoPath())
 			}
-			if stdout, err := git.NewCommand(args...).
-				SetDescription(fmt.Sprintf("Repository Garbage Collection: %s", repo.FullName())).
-				RunInDirTimeout(
-					time.Duration(setting.Git.Timeout.GC)*time.Second,
-					repo.RepoPath()); err != nil {
+
+			if err != nil {
 				log.Error("Repository garbage collection failed for %v. Stdout: %s\nError: %v", repo, stdout, err)
-				return fmt.Errorf("Repository garbage collection failed: Error: %v", err)
+				desc := fmt.Sprintf("Repository garbage collection failed for %s. Stdout: %s\nError: %v", repo.RepoPath(), stdout, err)
+				if err = models.CreateRepositoryNotice(desc); err != nil {
+					log.Error("CreateRepositoryNotice: %v", err)
+				}
+				return fmt.Errorf("Repository garbage collection failed in repo: %s: Error: %v", repo.FullName(), err)
 			}
 			return nil
 		},
@@ -90,7 +101,7 @@ func GitGcRepos(ctx context.Context) error {
 	return nil
 }
 
-func gatherMissingRepoRecords() ([]*models.Repository, error) {
+func gatherMissingRepoRecords(ctx context.Context) ([]*models.Repository, error) {
 	repos := make([]*models.Repository, 0, 10)
 	if err := models.Iterate(
 		models.DefaultDBContext(),
@@ -98,24 +109,33 @@ func gatherMissingRepoRecords() ([]*models.Repository, error) {
 		builder.Gt{"id": 0},
 		func(idx int, bean interface{}) error {
 			repo := bean.(*models.Repository)
+			select {
+			case <-ctx.Done():
+				return models.ErrCancelledf("during gathering missing repo records before checking %s", repo.FullName())
+			default:
+			}
 			if !com.IsDir(repo.RepoPath()) {
 				repos = append(repos, repo)
 			}
 			return nil
 		},
 	); err != nil {
-		if err2 := models.CreateRepositoryNotice(fmt.Sprintf("gatherMissingRepoRecords: %v", err)); err2 != nil {
-			return nil, fmt.Errorf("CreateRepositoryNotice: %v", err)
+		if strings.HasPrefix("Aborted gathering missing repo", err.Error()) {
+			return nil, err
+		}
+		if err2 := models.CreateRepositoryNotice("gatherMissingRepoRecords: %v", err); err2 != nil {
+			log.Error("CreateRepositoryNotice: %v", err2)
 		}
+		return nil, err
 	}
 	return repos, nil
 }
 
 // DeleteMissingRepositories deletes all repository records that lost Git files.
-func DeleteMissingRepositories(doer *models.User) error {
-	repos, err := gatherMissingRepoRecords()
+func DeleteMissingRepositories(ctx context.Context, doer *models.User) error {
+	repos, err := gatherMissingRepoRecords(ctx)
 	if err != nil {
-		return fmt.Errorf("gatherMissingRepoRecords: %v", err)
+		return err
 	}
 
 	if len(repos) == 0 {
@@ -123,10 +143,16 @@ func DeleteMissingRepositories(doer *models.User) error {
 	}
 
 	for _, repo := range repos {
+		select {
+		case <-ctx.Done():
+			return models.ErrCancelledf("during DeleteMissingRepositories before %s", repo.FullName())
+		default:
+		}
 		log.Trace("Deleting %d/%d...", repo.OwnerID, repo.ID)
 		if err := models.DeleteRepository(doer, repo.OwnerID, repo.ID); err != nil {
-			if err2 := models.CreateRepositoryNotice(fmt.Sprintf("DeleteRepository [%d]: %v", repo.ID, err)); err2 != nil {
-				return fmt.Errorf("CreateRepositoryNotice: %v", err)
+			log.Error("Failed to DeleteRepository %s [%d]: Error: %v", repo.FullName(), repo.ID, err)
+			if err2 := models.CreateRepositoryNotice("Failed to DeleteRepository %s [%d]: Error: %v", repo.FullName(), repo.ID, err); err2 != nil {
+				log.Error("CreateRepositoryNotice: %v", err)
 			}
 		}
 	}
@@ -134,10 +160,10 @@ func DeleteMissingRepositories(doer *models.User) error {
 }
 
 // ReinitMissingRepositories reinitializes all repository records that lost Git files.
-func ReinitMissingRepositories() error {
-	repos, err := gatherMissingRepoRecords()
+func ReinitMissingRepositories(ctx context.Context) error {
+	repos, err := gatherMissingRepoRecords(ctx)
 	if err != nil {
-		return fmt.Errorf("gatherMissingRepoRecords: %v", err)
+		return err
 	}
 
 	if len(repos) == 0 {
@@ -145,10 +171,16 @@ func ReinitMissingRepositories() error {
 	}
 
 	for _, repo := range repos {
+		select {
+		case <-ctx.Done():
+			return models.ErrCancelledf("during ReinitMissingRepositories before %s", repo.FullName())
+		default:
+		}
 		log.Trace("Initializing %d/%d...", repo.OwnerID, repo.ID)
 		if err := git.InitRepository(repo.RepoPath(), true); err != nil {
-			if err2 := models.CreateRepositoryNotice(fmt.Sprintf("InitRepository [%d]: %v", repo.ID, err)); err2 != nil {
-				return fmt.Errorf("CreateRepositoryNotice: %v", err)
+			log.Error("Unable (re)initialize repository %d at %s. Error: %v", repo.ID, repo.RepoPath(), err)
+			if err2 := models.CreateRepositoryNotice("InitRepository [%d]: %v", repo.ID, err); err2 != nil {
+				log.Error("CreateRepositoryNotice: %v", err2)
 			}
 		}
 	}
diff --git a/modules/repository/hooks.go b/modules/repository/hooks.go
index 94f570bf3a..6050f21f7f 100644
--- a/modules/repository/hooks.go
+++ b/modules/repository/hooks.go
@@ -160,17 +160,18 @@ func SyncRepositoryHooks(ctx context.Context) error {
 		new(models.Repository),
 		builder.Gt{"id": 0},
 		func(idx int, bean interface{}) error {
+			repo := bean.(*models.Repository)
 			select {
 			case <-ctx.Done():
-				return fmt.Errorf("Aborted due to shutdown")
+				return models.ErrCancelledf("before sync repository hooks for %s", repo.FullName())
 			default:
 			}
 
-			if err := createDelegateHooks(bean.(*models.Repository).RepoPath()); err != nil {
+			if err := createDelegateHooks(repo.RepoPath()); err != nil {
 				return fmt.Errorf("SyncRepositoryHook: %v", err)
 			}
-			if bean.(*models.Repository).HasWiki() {
-				if err := createDelegateHooks(bean.(*models.Repository).WikiPath()); err != nil {
+			if repo.HasWiki() {
+				if err := createDelegateHooks(repo.WikiPath()); err != nil {
 					return fmt.Errorf("SyncRepositoryHook: %v", err)
 				}
 			}
diff --git a/modules/setting/cron.go b/modules/setting/cron.go
index 77f55168aa..c8228ddaa8 100644
--- a/modules/setting/cron.go
+++ b/modules/setting/cron.go
@@ -4,129 +4,8 @@
 
 package setting
 
-import (
-	"time"
-
-	"code.gitea.io/gitea/modules/log"
-)
-
-var (
-
-	// Cron tasks
-	Cron = struct {
-		UpdateMirror struct {
-			Enabled    bool
-			RunAtStart bool
-			Schedule   string
-		} `ini:"cron.update_mirrors"`
-		RepoHealthCheck struct {
-			Enabled    bool
-			RunAtStart bool
-			Schedule   string
-			Timeout    time.Duration
-			Args       []string `delim:" "`
-		} `ini:"cron.repo_health_check"`
-		CheckRepoStats struct {
-			Enabled    bool
-			RunAtStart bool
-			Schedule   string
-		} `ini:"cron.check_repo_stats"`
-		ArchiveCleanup struct {
-			Enabled    bool
-			RunAtStart bool
-			Schedule   string
-			OlderThan  time.Duration
-		} `ini:"cron.archive_cleanup"`
-		SyncExternalUsers struct {
-			Enabled        bool
-			RunAtStart     bool
-			Schedule       string
-			UpdateExisting bool
-		} `ini:"cron.sync_external_users"`
-		DeletedBranchesCleanup struct {
-			Enabled    bool
-			RunAtStart bool
-			Schedule   string
-			OlderThan  time.Duration
-		} `ini:"cron.deleted_branches_cleanup"`
-		UpdateMigrationPosterID struct {
-			Schedule string
-		} `ini:"cron.update_migration_poster_id"`
-	}{
-		UpdateMirror: struct {
-			Enabled    bool
-			RunAtStart bool
-			Schedule   string
-		}{
-			Enabled:    true,
-			RunAtStart: false,
-			Schedule:   "@every 10m",
-		},
-		RepoHealthCheck: struct {
-			Enabled    bool
-			RunAtStart bool
-			Schedule   string
-			Timeout    time.Duration
-			Args       []string `delim:" "`
-		}{
-			Enabled:    true,
-			RunAtStart: false,
-			Schedule:   "@every 24h",
-			Timeout:    60 * time.Second,
-			Args:       []string{},
-		},
-		CheckRepoStats: struct {
-			Enabled    bool
-			RunAtStart bool
-			Schedule   string
-		}{
-			Enabled:    true,
-			RunAtStart: true,
-			Schedule:   "@every 24h",
-		},
-		ArchiveCleanup: struct {
-			Enabled    bool
-			RunAtStart bool
-			Schedule   string
-			OlderThan  time.Duration
-		}{
-			Enabled:    true,
-			RunAtStart: true,
-			Schedule:   "@every 24h",
-			OlderThan:  24 * time.Hour,
-		},
-		SyncExternalUsers: struct {
-			Enabled        bool
-			RunAtStart     bool
-			Schedule       string
-			UpdateExisting bool
-		}{
-			Enabled:        true,
-			RunAtStart:     false,
-			Schedule:       "@every 24h",
-			UpdateExisting: true,
-		},
-		DeletedBranchesCleanup: struct {
-			Enabled    bool
-			RunAtStart bool
-			Schedule   string
-			OlderThan  time.Duration
-		}{
-			Enabled:    true,
-			RunAtStart: true,
-			Schedule:   "@every 24h",
-			OlderThan:  24 * time.Hour,
-		},
-		UpdateMigrationPosterID: struct {
-			Schedule string
-		}{
-			Schedule: "@every 24h",
-		},
-	}
-)
-
-func newCron() {
-	if err := Cfg.Section("cron").MapTo(&Cron); err != nil {
-		log.Fatal("Failed to map Cron settings: %v", err)
-	}
+// GetCronSettings maps the cron subsection to the provided config
+func GetCronSettings(name string, config interface{}) (interface{}, error) {
+	err := Cfg.Section("cron." + name).MapTo(config)
+	return config, err
 }
diff --git a/modules/setting/setting.go b/modules/setting/setting.go
index dd7dbd3fdf..ede4687c81 100644
--- a/modules/setting/setting.go
+++ b/modules/setting/setting.go
@@ -981,7 +981,6 @@ func NewContext() {
 	u.Path = path.Join(u.Path, "api", "swagger")
 	API.SwaggerURL = u.String()
 
-	newCron()
 	newGit()
 
 	sec = Cfg.Section("mirror")