diff options
author | FuXiaoHei <fuxiaohei@vip.qq.com> | 2023-05-19 21:37:57 +0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-05-19 21:37:57 +0800 |
commit | c757765a9e5c2d4f73b1a7c3debe3548c735bd54 (patch) | |
tree | ffed7692321760ea4f9c72670ed31437b52ff0e0 /models | |
parent | 7985cde84df5ee93bfb37b20681d69e67d3f32fc (diff) | |
download | gitea-c757765a9e5c2d4f73b1a7c3debe3548c735bd54.tar.gz gitea-c757765a9e5c2d4f73b1a7c3debe3548c735bd54.zip |
Implement actions artifacts (#22738)
Implement action artifacts server api.
This change is used for supporting
https://github.com/actions/upload-artifact and
https://github.com/actions/download-artifact in gitea actions. It can
run sample workflow from doc
https://docs.github.com/en/actions/using-workflows/storing-workflow-data-as-artifacts.
The api design is inspired by
https://github.com/nektos/act/blob/master/pkg/artifacts/server.go and
includes some changes from gitea internal structs and methods.
Actions artifacts contains two parts:
- Gitea server api and storage (this pr implement basic design without
some complex cases supports)
- Runner communicate with gitea server api (in comming)
Old pr https://github.com/go-gitea/gitea/pull/22345 is outdated after
actions merged. I create new pr from main branch.
![897f7694-3e0f-4f7c-bb4b-9936624ead45](https://user-images.githubusercontent.com/2142787/219382371-eb3cf810-e4e0-456b-a8ff-aecc2b1a1032.jpeg)
Add artifacts list in actions workflow page.
Diffstat (limited to 'models')
-rw-r--r-- | models/actions/artifact.go | 122 | ||||
-rw-r--r-- | models/fixtures/access_token.yml | 2 | ||||
-rw-r--r-- | models/fixtures/action_run.yml | 19 | ||||
-rw-r--r-- | models/fixtures/action_run_job.yml | 14 | ||||
-rw-r--r-- | models/fixtures/action_task.yml | 20 | ||||
-rw-r--r-- | models/migrations/migrations.go | 2 | ||||
-rw-r--r-- | models/migrations/v1_20/v257.go | 33 | ||||
-rw-r--r-- | models/repo.go | 15 | ||||
-rw-r--r-- | models/unittest/testdb.go | 2 |
9 files changed, 227 insertions, 2 deletions
diff --git a/models/actions/artifact.go b/models/actions/artifact.go new file mode 100644 index 0000000000..1b45fce067 --- /dev/null +++ b/models/actions/artifact.go @@ -0,0 +1,122 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +// This artifact server is inspired by https://github.com/nektos/act/blob/master/pkg/artifacts/server.go. +// It updates url setting and uses ObjectStore to handle artifacts persistence. + +package actions + +import ( + "context" + "errors" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" +) + +const ( + // ArtifactStatusUploadPending is the status of an artifact upload that is pending + ArtifactStatusUploadPending = 1 + // ArtifactStatusUploadConfirmed is the status of an artifact upload that is confirmed + ArtifactStatusUploadConfirmed = 2 + // ArtifactStatusUploadError is the status of an artifact upload that is errored + ArtifactStatusUploadError = 3 +) + +func init() { + db.RegisterModel(new(ActionArtifact)) +} + +// ActionArtifact is a file that is stored in the artifact storage. +type ActionArtifact struct { + ID int64 `xorm:"pk autoincr"` + RunID int64 `xorm:"index UNIQUE(runid_name)"` // The run id of the artifact + RunnerID int64 + RepoID int64 `xorm:"index"` + OwnerID int64 + CommitSHA string + StoragePath string // The path to the artifact in the storage + FileSize int64 // The size of the artifact in bytes + FileCompressedSize int64 // The size of the artifact in bytes after gzip compression + ContentEncoding string // The content encoding of the artifact + ArtifactPath string // The path to the artifact when runner uploads it + ArtifactName string `xorm:"UNIQUE(runid_name)"` // The name of the artifact when runner uploads it + Status int64 `xorm:"index"` // The status of the artifact, uploading, expired or need-delete + CreatedUnix timeutil.TimeStamp `xorm:"created"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated index"` +} + +// CreateArtifact create a new artifact with task info or get same named artifact in the same run +func CreateArtifact(ctx context.Context, t *ActionTask, artifactName string) (*ActionArtifact, error) { + if err := t.LoadJob(ctx); err != nil { + return nil, err + } + artifact, err := getArtifactByArtifactName(ctx, t.Job.RunID, artifactName) + if errors.Is(err, util.ErrNotExist) { + artifact := &ActionArtifact{ + RunID: t.Job.RunID, + RunnerID: t.RunnerID, + RepoID: t.RepoID, + OwnerID: t.OwnerID, + CommitSHA: t.CommitSHA, + Status: ArtifactStatusUploadPending, + } + if _, err := db.GetEngine(ctx).Insert(artifact); err != nil { + return nil, err + } + return artifact, nil + } else if err != nil { + return nil, err + } + return artifact, nil +} + +func getArtifactByArtifactName(ctx context.Context, runID int64, name string) (*ActionArtifact, error) { + var art ActionArtifact + has, err := db.GetEngine(ctx).Where("run_id = ? AND artifact_name = ?", runID, name).Get(&art) + if err != nil { + return nil, err + } else if !has { + return nil, util.ErrNotExist + } + return &art, nil +} + +// GetArtifactByID returns an artifact by id +func GetArtifactByID(ctx context.Context, id int64) (*ActionArtifact, error) { + var art ActionArtifact + has, err := db.GetEngine(ctx).ID(id).Get(&art) + if err != nil { + return nil, err + } else if !has { + return nil, util.ErrNotExist + } + + return &art, nil +} + +// UpdateArtifactByID updates an artifact by id +func UpdateArtifactByID(ctx context.Context, id int64, art *ActionArtifact) error { + art.ID = id + _, err := db.GetEngine(ctx).ID(id).AllCols().Update(art) + return err +} + +// ListArtifactsByRunID returns all artifacts of a run +func ListArtifactsByRunID(ctx context.Context, runID int64) ([]*ActionArtifact, error) { + arts := make([]*ActionArtifact, 0, 10) + return arts, db.GetEngine(ctx).Where("run_id=?", runID).Find(&arts) +} + +// ListUploadedArtifactsByRunID returns all uploaded artifacts of a run +func ListUploadedArtifactsByRunID(ctx context.Context, runID int64) ([]*ActionArtifact, error) { + arts := make([]*ActionArtifact, 0, 10) + return arts, db.GetEngine(ctx).Where("run_id=? AND status=?", runID, ArtifactStatusUploadConfirmed).Find(&arts) +} + +// ListArtifactsByRepoID returns all artifacts of a repo +func ListArtifactsByRepoID(ctx context.Context, repoID int64) ([]*ActionArtifact, error) { + arts := make([]*ActionArtifact, 0, 10) + return arts, db.GetEngine(ctx).Where("repo_id=?", repoID).Find(&arts) +} diff --git a/models/fixtures/access_token.yml b/models/fixtures/access_token.yml index 5ff5d66f66..791b3da1c4 100644 --- a/models/fixtures/access_token.yml +++ b/models/fixtures/access_token.yml @@ -30,4 +30,4 @@ token_last_eight: 69d28c91 created_unix: 946687980 updated_unix: 946687980 -#commented out tokens so you can see what they are in plaintext
\ No newline at end of file +#commented out tokens so you can see what they are in plaintext diff --git a/models/fixtures/action_run.yml b/models/fixtures/action_run.yml new file mode 100644 index 0000000000..2c2151f354 --- /dev/null +++ b/models/fixtures/action_run.yml @@ -0,0 +1,19 @@ +- + id: 791 + title: "update actions" + repo_id: 4 + owner_id: 1 + workflow_id: "artifact.yaml" + index: 187 + trigger_user_id: 1 + ref: "refs/heads/master" + commit_sha: "c2d72f548424103f01ee1dc02889c1e2bff816b0" + event: "push" + is_fork_pull_request: 0 + status: 1 + started: 1683636528 + stopped: 1683636626 + created: 1683636108 + updated: 1683636626 + need_approval: 0 + approved_by: 0 diff --git a/models/fixtures/action_run_job.yml b/models/fixtures/action_run_job.yml new file mode 100644 index 0000000000..071998b979 --- /dev/null +++ b/models/fixtures/action_run_job.yml @@ -0,0 +1,14 @@ +- + id: 192 + run_id: 791 + repo_id: 4 + owner_id: 1 + commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0 + is_fork_pull_request: 0 + name: job_2 + attempt: 1 + job_id: job_2 + task_id: 47 + status: 1 + started: 1683636528 + stopped: 1683636626 diff --git a/models/fixtures/action_task.yml b/models/fixtures/action_task.yml new file mode 100644 index 0000000000..c78fb3c5d6 --- /dev/null +++ b/models/fixtures/action_task.yml @@ -0,0 +1,20 @@ +- + id: 47 + job_id: 192 + attempt: 3 + runner_id: 1 + status: 6 # 6 is the status code for "running", running task can upload artifacts + started: 1683636528 + stopped: 1683636626 + repo_id: 4 + owner_id: 1 + commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0 + is_fork_pull_request: 0 + token_hash: 6d8ef48297195edcc8e22c70b3020eaa06c52976db67d39b4260c64a69a2cc1508825121b7b8394e48e00b1bf8718b2a867e + token_salt: jVuKnSPGgy + token_last_eight: eeb1a71a + log_filename: artifact-test2/2f/47.log + log_in_storage: 1 + log_length: 707 + log_size: 90179 + log_expired: 0 diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 0e84ae9f0e..49bc0be4e5 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -491,6 +491,8 @@ var migrations = []Migration{ NewMigration("Add ArchivedUnix Column", v1_20.AddArchivedUnixToRepository), // v256 -> v257 NewMigration("Add is_internal column to package", v1_20.AddIsInternalColumnToPackage), + // v257 -> v258 + NewMigration("Add Actions Artifact table", v1_20.CreateActionArtifactTable), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v1_20/v257.go b/models/migrations/v1_20/v257.go new file mode 100644 index 0000000000..6c6ca4c748 --- /dev/null +++ b/models/migrations/v1_20/v257.go @@ -0,0 +1,33 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package v1_20 //nolint + +import ( + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/xorm" +) + +func CreateActionArtifactTable(x *xorm.Engine) error { + // ActionArtifact is a file that is stored in the artifact storage. + type ActionArtifact struct { + ID int64 `xorm:"pk autoincr"` + RunID int64 `xorm:"index UNIQUE(runid_name)"` // The run id of the artifact + RunnerID int64 + RepoID int64 `xorm:"index"` + OwnerID int64 + CommitSHA string + StoragePath string // The path to the artifact in the storage + FileSize int64 // The size of the artifact in bytes + FileCompressedSize int64 // The size of the artifact in bytes after gzip compression + ContentEncoding string // The content encoding of the artifact + ArtifactPath string // The path to the artifact when runner uploads it + ArtifactName string `xorm:"UNIQUE(runid_name)"` // The name of the artifact when runner uploads it + Status int64 `xorm:"index"` // The status of the artifact + CreatedUnix timeutil.TimeStamp `xorm:"created"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated index"` + } + + return x.Sync(new(ActionArtifact)) +} diff --git a/models/repo.go b/models/repo.go index 5903132862..2e0e8af16c 100644 --- a/models/repo.go +++ b/models/repo.go @@ -59,6 +59,12 @@ func DeleteRepository(doer *user_model.User, uid, repoID int64) error { return fmt.Errorf("find actions tasks of repo %v: %w", repoID, err) } + // Query the artifacts of this repo, they will be needed after they have been deleted to remove artifacts files in ObjectStorage + artifacts, err := actions_model.ListArtifactsByRepoID(ctx, repoID) + if err != nil { + return fmt.Errorf("list actions artifacts of repo %v: %w", repoID, err) + } + // In case is a organization. org, err := user_model.GetUserByID(ctx, uid) if err != nil { @@ -164,6 +170,7 @@ func DeleteRepository(doer *user_model.User, uid, repoID int64) error { &actions_model.ActionRunJob{RepoID: repoID}, &actions_model.ActionRun{RepoID: repoID}, &actions_model.ActionRunner{RepoID: repoID}, + &actions_model.ActionArtifact{RepoID: repoID}, ); err != nil { return fmt.Errorf("deleteBeans: %w", err) } @@ -336,6 +343,14 @@ func DeleteRepository(doer *user_model.User, uid, repoID int64) error { } } + // delete actions artifacts in ObjectStorage after the repo have already been deleted + for _, art := range artifacts { + if err := storage.ActionsArtifacts.Delete(art.StoragePath); err != nil { + log.Error("remove artifact file %q: %v", art.StoragePath, err) + // go on + } + } + return nil } diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index 10a70ad9f8..5351ff1139 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -126,7 +126,7 @@ func MainTest(m *testing.M, testOpts *TestOptions) { setting.Packages.Storage.Path = filepath.Join(setting.AppDataPath, "packages") - setting.Actions.Storage.Path = filepath.Join(setting.AppDataPath, "actions_log") + setting.Actions.LogStorage.Path = filepath.Join(setting.AppDataPath, "actions_log") setting.Git.HomePath = filepath.Join(setting.AppDataPath, "home") |