Make Actions tasks/jobs timeouts configurable by the user (#27400) (#27402)

Backport #27400 by @fantognazza

With this PR we added the possibility to configure the Actions timeouts
values for killing tasks/jobs.
Particularly this enhancement is closely related to the `act_runner`
configuration reported below:
```
# The timeout for a job to be finished.
# Please note that the Gitea instance also has a timeout (3h by default) for the job.
# So the job could be stopped by the Gitea instance if it's timeout is shorter than this.
timeout: 3h
```

---

Setting the corresponding key in the INI configuration file, it is
possible to let jobs run for more than 3 hours.

Signed-off-by: Francesco Antognazza <francesco.antognazza@gmail.com>

diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini
index cd6b2a914d2373805de29fcb0779c4ad42d5dd0a..7db3c157c9dca9362fbc0d489fa3bae91ce234d3 100644 (file)
--- a/custom/conf/app.example.ini
+++ b/custom/conf/app.example.ini
@@ -2568,6 +2568,12 @@ LEVEL = Info
 ;DEFAULT_ACTIONS_URL = github
 ;; Default artifact retention time in days, default is 90 days
 ;ARTIFACT_RETENTION_DAYS = 90
+;; Timeout to stop the task which have running status, but haven't been updated for a long time
+;ZOMBIE_TASK_TIMEOUT = 10m
+;; Timeout to stop the tasks which have running status and continuous updates, but don't end for a long time
+;ENDLESS_TASK_TIMEOUT = 3h
+;; Timeout to cancel the jobs which have waiting status, but haven't been picked by a runner for a long time
+;ABANDONED_JOB_TIMEOUT = 24h
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

diff --git a/docs/content/administration/config-cheat-sheet.en-us.md b/docs/content/administration/config-cheat-sheet.en-us.md
index aa29f8be4b8ca92034dcfeb843cacbf3ea0913f5..a337ae82a05be845c48fc775178f14ad2ad4535e 100644 (file)
--- a/docs/content/administration/config-cheat-sheet.en-us.md
+++ b/docs/content/administration/config-cheat-sheet.en-us.md
@@ -1389,6 +1389,9 @@ PROXY_HOSTS = *.github.com
 - `STORAGE_TYPE`: **local**: Storage type for actions logs, `local` for local disk or `minio` for s3 compatible object storage service, default is `local` or other name defined with `[storage.xxx]`
 - `MINIO_BASE_PATH`: **actions_log/**: Minio base path on the bucket only available when STORAGE_TYPE is `minio`
 - `ARTIFACT_RETENTION_DAYS`: **90**: Number of days to keep artifacts. Set to 0 to disable artifact retention. Default is 90 days if not set.
+- `ZOMBIE_TASK_TIMEOUT`: **10m**: Timeout to stop the task which have running status, but haven't been updated for a long time
+- `ENDLESS_TASK_TIMEOUT`: **3h**: Timeout to stop the tasks which have running status and continuous updates, but don't end for a long time
+- `ABANDONED_JOB_TIMEOUT`: **24h**: Timeout to cancel the jobs which have waiting status, but haven't been picked by a runner for a long time
 
 `DEFAULT_ACTIONS_URL` indicates where the Gitea Actions runners should find the actions with relative path.
 For example, `uses: actions/checkout@v3` means `https://github.com/actions/checkout@v3` since the value of `DEFAULT_ACTIONS_URL` is `github`.

diff --git a/modules/setting/actions.go b/modules/setting/actions.go
index 28c84f01491aa66d00dffe86eba1cf84f59f5ecb..026bab4bfcfdcb5351e4a3762af6792b1ce6cc36 100644 (file)
--- a/modules/setting/actions.go
+++ b/modules/setting/actions.go
@@ -6,6 +6,7 @@ package setting
 import (
        "fmt"
        "strings"
+       "time"
 
        "code.gitea.io/gitea/modules/log"
 )
@@ -18,6 +19,9 @@ var (
                ArtifactRetentionDays int64    `ini:"ARTIFACT_RETENTION_DAYS"`
                Enabled               bool
                DefaultActionsURL     defaultActionsURL `ini:"DEFAULT_ACTIONS_URL"`
+               ZombieTaskTimeout     time.Duration     `ini:"ZOMBIE_TASK_TIMEOUT"`
+               EndlessTaskTimeout    time.Duration     `ini:"ENDLESS_TASK_TIMEOUT"`
+               AbandonedJobTimeout   time.Duration     `ini:"ABANDONED_JOB_TIMEOUT"`
        }{
                Enabled:           true,
                DefaultActionsURL: defaultActionsURLGitHub,
@@ -82,5 +86,9 @@ func loadActionsFrom(rootCfg ConfigProvider) error {
                Actions.ArtifactRetentionDays = 90
        }
 
+       Actions.ZombieTaskTimeout = sec.Key("ZOMBIE_TASK_TIMEOUT").MustDuration(10 * time.Minute)
+       Actions.EndlessTaskTimeout = sec.Key("ENDLESS_TASK_TIMEOUT").MustDuration(3 * time.Hour)
+       Actions.AbandonedJobTimeout = sec.Key("ABANDONED_JOB_TIMEOUT").MustDuration(24 * time.Hour)
+
        return err
 }

diff --git a/services/actions/clear_tasks.go b/services/actions/clear_tasks.go
index d2893e4f23e020e382906d434585a351f37ff1ea..7c7043c42f4b8375c70d81e6ed85f0c29eabad22 100644 (file)
--- a/services/actions/clear_tasks.go
+++ b/services/actions/clear_tasks.go
@@ -12,20 +12,15 @@ import (
        "code.gitea.io/gitea/models/db"
        "code.gitea.io/gitea/modules/actions"
        "code.gitea.io/gitea/modules/log"
+       "code.gitea.io/gitea/modules/setting"
        "code.gitea.io/gitea/modules/timeutil"
 )
 
-const (
-       zombieTaskTimeout   = 10 * time.Minute
-       endlessTaskTimeout  = 3 * time.Hour
-       abandonedJobTimeout = 24 * time.Hour
-)
-
 // StopZombieTasks stops the task which have running status, but haven't been updated for a long time
 func StopZombieTasks(ctx context.Context) error {
        return stopTasks(ctx, actions_model.FindTaskOptions{
                Status:        actions_model.StatusRunning,
-               UpdatedBefore: timeutil.TimeStamp(time.Now().Add(-zombieTaskTimeout).Unix()),
+               UpdatedBefore: timeutil.TimeStamp(time.Now().Add(-setting.Actions.ZombieTaskTimeout).Unix()),
        })
 }
 
@@ -33,7 +28,7 @@ func StopZombieTasks(ctx context.Context) error {
 func StopEndlessTasks(ctx context.Context) error {
        return stopTasks(ctx, actions_model.FindTaskOptions{
                Status:        actions_model.StatusRunning,
-               StartedBefore: timeutil.TimeStamp(time.Now().Add(-endlessTaskTimeout).Unix()),
+               StartedBefore: timeutil.TimeStamp(time.Now().Add(-setting.Actions.EndlessTaskTimeout).Unix()),
        })
 }
 
@@ -81,7 +76,7 @@ func stopTasks(ctx context.Context, opts actions_model.FindTaskOptions) error {
 func CancelAbandonedJobs(ctx context.Context) error {
        jobs, _, err := actions_model.FindRunJobs(ctx, actions_model.FindRunJobOptions{
                Statuses:      []actions_model.Status{actions_model.StatusWaiting, actions_model.StatusBlocked},
-               UpdatedBefore: timeutil.TimeStamp(time.Now().Add(-abandonedJobTimeout).Unix()),
+               UpdatedBefore: timeutil.TimeStamp(time.Now().Add(-setting.Actions.AbandonedJobTimeout).Unix()),
        })
        if err != nil {
                log.Warn("find abandoned tasks: %v", err)