123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498 |
- // Copyright 2023 The Gitea Authors. All rights reserved.
- // SPDX-License-Identifier: MIT
-
- package actions
-
- // GitHub Actions Artifacts API Simple Description
- //
- // 1. Upload artifact
- // 1.1. Post upload url
- // Post: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts?api-version=6.0-preview
- // Request:
- // {
- // "Type": "actions_storage",
- // "Name": "artifact"
- // }
- // Response:
- // {
- // "fileContainerResourceUrl":"/api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/upload"
- // }
- // it acquires an upload url for artifact upload
- // 1.2. Upload artifact
- // PUT: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/upload?itemPath=artifact%2Ffilename
- // it upload chunk with headers:
- // x-tfs-filelength: 1024 // total file length
- // content-length: 1024 // chunk length
- // x-actions-results-md5: md5sum // md5sum of chunk
- // content-range: bytes 0-1023/1024 // chunk range
- // we save all chunks to one storage directory after md5sum check
- // 1.3. Confirm upload
- // PATCH: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/upload?itemPath=artifact%2Ffilename
- // it confirm upload and merge all chunks to one file, save this file to storage
- //
- // 2. Download artifact
- // 2.1 list artifacts
- // GET: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts?api-version=6.0-preview
- // Response:
- // {
- // "count": 1,
- // "value": [
- // {
- // "name": "artifact",
- // "fileContainerResourceUrl": "/api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/path"
- // }
- // ]
- // }
- // 2.2 download artifact
- // GET: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/path?api-version=6.0-preview
- // Response:
- // {
- // "value": [
- // {
- // "contentLocation": "/api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/download",
- // "path": "artifact/filename",
- // "itemType": "file"
- // }
- // ]
- // }
- // 2.3 download artifact file
- // GET: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/download?itemPath=artifact%2Ffilename
- // Response:
- // download file
- //
-
- import (
- "crypto/md5"
- "errors"
- "fmt"
- "net/http"
- "strconv"
- "strings"
-
- "code.gitea.io/gitea/models/actions"
- "code.gitea.io/gitea/models/db"
- "code.gitea.io/gitea/modules/json"
- "code.gitea.io/gitea/modules/log"
- "code.gitea.io/gitea/modules/setting"
- "code.gitea.io/gitea/modules/storage"
- "code.gitea.io/gitea/modules/util"
- "code.gitea.io/gitea/modules/web"
- web_types "code.gitea.io/gitea/modules/web/types"
- actions_service "code.gitea.io/gitea/services/actions"
- "code.gitea.io/gitea/services/context"
- )
-
- const artifactRouteBase = "/_apis/pipelines/workflows/{run_id}/artifacts"
-
- type artifactContextKeyType struct{}
-
- var artifactContextKey = artifactContextKeyType{}
-
- type ArtifactContext struct {
- *context.Base
-
- ActionTask *actions.ActionTask
- }
-
- func init() {
- web.RegisterResponseStatusProvider[*ArtifactContext](func(req *http.Request) web_types.ResponseStatusProvider {
- return req.Context().Value(artifactContextKey).(*ArtifactContext)
- })
- }
-
- func ArtifactsRoutes(prefix string) *web.Route {
- m := web.NewRoute()
- m.Use(ArtifactContexter())
-
- r := artifactRoutes{
- prefix: prefix,
- fs: storage.ActionsArtifacts,
- }
-
- m.Group(artifactRouteBase, func() {
- // retrieve, list and confirm artifacts
- m.Combo("").Get(r.listArtifacts).Post(r.getUploadArtifactURL).Patch(r.comfirmUploadArtifact)
- // handle container artifacts list and download
- m.Put("/{artifact_hash}/upload", r.uploadArtifact)
- // handle artifacts download
- m.Get("/{artifact_hash}/download_url", r.getDownloadArtifactURL)
- m.Get("/{artifact_id}/download", r.downloadArtifact)
- })
-
- return m
- }
-
- func ArtifactContexter() func(next http.Handler) http.Handler {
- return func(next http.Handler) http.Handler {
- return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
- base, baseCleanUp := context.NewBaseContext(resp, req)
- defer baseCleanUp()
-
- ctx := &ArtifactContext{Base: base}
- ctx.AppendContextValue(artifactContextKey, ctx)
-
- // action task call server api with Bearer ACTIONS_RUNTIME_TOKEN
- // we should verify the ACTIONS_RUNTIME_TOKEN
- authHeader := req.Header.Get("Authorization")
- if len(authHeader) == 0 || !strings.HasPrefix(authHeader, "Bearer ") {
- ctx.Error(http.StatusUnauthorized, "Bad authorization header")
- return
- }
-
- // New act_runner uses jwt to authenticate
- tID, err := actions_service.ParseAuthorizationToken(req)
-
- var task *actions.ActionTask
- if err == nil {
-
- task, err = actions.GetTaskByID(req.Context(), tID)
- if err != nil {
- log.Error("Error runner api getting task by ID: %v", err)
- ctx.Error(http.StatusInternalServerError, "Error runner api getting task by ID")
- return
- }
- if task.Status != actions.StatusRunning {
- log.Error("Error runner api getting task: task is not running")
- ctx.Error(http.StatusInternalServerError, "Error runner api getting task: task is not running")
- return
- }
- } else {
- // Old act_runner uses GITEA_TOKEN to authenticate
- authToken := strings.TrimPrefix(authHeader, "Bearer ")
-
- task, err = actions.GetRunningTaskByToken(req.Context(), authToken)
- if err != nil {
- log.Error("Error runner api getting task: %v", err)
- ctx.Error(http.StatusInternalServerError, "Error runner api getting task")
- return
- }
- }
-
- if err := task.LoadJob(req.Context()); err != nil {
- log.Error("Error runner api getting job: %v", err)
- ctx.Error(http.StatusInternalServerError, "Error runner api getting job")
- return
- }
-
- ctx.ActionTask = task
- next.ServeHTTP(ctx.Resp, ctx.Req)
- })
- }
- }
-
- type artifactRoutes struct {
- prefix string
- fs storage.ObjectStorage
- }
-
- func (ar artifactRoutes) buildArtifactURL(runID int64, artifactHash, suffix string) string {
- uploadURL := strings.TrimSuffix(setting.AppURL, "/") + strings.TrimSuffix(ar.prefix, "/") +
- strings.ReplaceAll(artifactRouteBase, "{run_id}", strconv.FormatInt(runID, 10)) +
- "/" + artifactHash + "/" + suffix
- return uploadURL
- }
-
- type getUploadArtifactRequest struct {
- Type string
- Name string
- RetentionDays int64
- }
-
- type getUploadArtifactResponse struct {
- FileContainerResourceURL string `json:"fileContainerResourceUrl"`
- }
-
- // getUploadArtifactURL generates a URL for uploading an artifact
- func (ar artifactRoutes) getUploadArtifactURL(ctx *ArtifactContext) {
- _, runID, ok := validateRunID(ctx)
- if !ok {
- return
- }
-
- var req getUploadArtifactRequest
- if err := json.NewDecoder(ctx.Req.Body).Decode(&req); err != nil {
- log.Error("Error decode request body: %v", err)
- ctx.Error(http.StatusInternalServerError, "Error decode request body")
- return
- }
-
- // set retention days
- retentionQuery := ""
- if req.RetentionDays > 0 {
- retentionQuery = fmt.Sprintf("?retentionDays=%d", req.RetentionDays)
- }
-
- // use md5(artifact_name) to create upload url
- artifactHash := fmt.Sprintf("%x", md5.Sum([]byte(req.Name)))
- resp := getUploadArtifactResponse{
- FileContainerResourceURL: ar.buildArtifactURL(runID, artifactHash, "upload"+retentionQuery),
- }
- log.Debug("[artifact] get upload url: %s", resp.FileContainerResourceURL)
- ctx.JSON(http.StatusOK, resp)
- }
-
- func (ar artifactRoutes) uploadArtifact(ctx *ArtifactContext) {
- task, runID, ok := validateRunID(ctx)
- if !ok {
- return
- }
- artifactName, artifactPath, ok := parseArtifactItemPath(ctx)
- if !ok {
- return
- }
-
- // get upload file size
- fileRealTotalSize, contentLength, err := getUploadFileSize(ctx)
- if err != nil {
- log.Error("Error get upload file size: %v", err)
- ctx.Error(http.StatusInternalServerError, "Error get upload file size")
- return
- }
-
- // get artifact retention days
- expiredDays := setting.Actions.ArtifactRetentionDays
- if queryRetentionDays := ctx.Req.URL.Query().Get("retentionDays"); queryRetentionDays != "" {
- expiredDays, err = strconv.ParseInt(queryRetentionDays, 10, 64)
- if err != nil {
- log.Error("Error parse retention days: %v", err)
- ctx.Error(http.StatusBadRequest, "Error parse retention days")
- return
- }
- }
- log.Debug("[artifact] upload chunk, name: %s, path: %s, size: %d, retention days: %d",
- artifactName, artifactPath, fileRealTotalSize, expiredDays)
-
- // create or get artifact with name and path
- artifact, err := actions.CreateArtifact(ctx, task, artifactName, artifactPath, expiredDays)
- if err != nil {
- log.Error("Error create or get artifact: %v", err)
- ctx.Error(http.StatusInternalServerError, "Error create or get artifact")
- return
- }
-
- // save chunk to storage, if success, return chunk stotal size
- // if artifact is not gzip when uploading, chunksTotalSize == fileRealTotalSize
- // if artifact is gzip when uploading, chunksTotalSize < fileRealTotalSize
- chunksTotalSize, err := saveUploadChunk(ar.fs, ctx, artifact, contentLength, runID)
- if err != nil {
- log.Error("Error save upload chunk: %v", err)
- ctx.Error(http.StatusInternalServerError, "Error save upload chunk")
- return
- }
-
- // update artifact size if zero or not match, over write artifact size
- if artifact.FileSize == 0 ||
- artifact.FileCompressedSize == 0 ||
- artifact.FileSize != fileRealTotalSize ||
- artifact.FileCompressedSize != chunksTotalSize {
- artifact.FileSize = fileRealTotalSize
- artifact.FileCompressedSize = chunksTotalSize
- artifact.ContentEncoding = ctx.Req.Header.Get("Content-Encoding")
- if err := actions.UpdateArtifactByID(ctx, artifact.ID, artifact); err != nil {
- log.Error("Error update artifact: %v", err)
- ctx.Error(http.StatusInternalServerError, "Error update artifact")
- return
- }
- log.Debug("[artifact] update artifact size, artifact_id: %d, size: %d, compressed size: %d",
- artifact.ID, artifact.FileSize, artifact.FileCompressedSize)
- }
-
- ctx.JSON(http.StatusOK, map[string]string{
- "message": "success",
- })
- }
-
- // comfirmUploadArtifact comfirm upload artifact.
- // if all chunks are uploaded, merge them to one file.
- func (ar artifactRoutes) comfirmUploadArtifact(ctx *ArtifactContext) {
- _, runID, ok := validateRunID(ctx)
- if !ok {
- return
- }
- artifactName := ctx.Req.URL.Query().Get("artifactName")
- if artifactName == "" {
- log.Error("Error artifact name is empty")
- ctx.Error(http.StatusBadRequest, "Error artifact name is empty")
- return
- }
- if err := mergeChunksForRun(ctx, ar.fs, runID, artifactName); err != nil {
- log.Error("Error merge chunks: %v", err)
- ctx.Error(http.StatusInternalServerError, "Error merge chunks")
- return
- }
- ctx.JSON(http.StatusOK, map[string]string{
- "message": "success",
- })
- }
-
- type (
- listArtifactsResponse struct {
- Count int64 `json:"count"`
- Value []listArtifactsResponseItem `json:"value"`
- }
- listArtifactsResponseItem struct {
- Name string `json:"name"`
- FileContainerResourceURL string `json:"fileContainerResourceUrl"`
- }
- )
-
- func (ar artifactRoutes) listArtifacts(ctx *ArtifactContext) {
- _, runID, ok := validateRunID(ctx)
- if !ok {
- return
- }
-
- artifacts, err := db.Find[actions.ActionArtifact](ctx, actions.FindArtifactsOptions{RunID: runID})
- if err != nil {
- log.Error("Error getting artifacts: %v", err)
- ctx.Error(http.StatusInternalServerError, err.Error())
- return
- }
- if len(artifacts) == 0 {
- log.Debug("[artifact] handleListArtifacts, no artifacts")
- ctx.Error(http.StatusNotFound)
- return
- }
-
- var (
- items []listArtifactsResponseItem
- values = make(map[string]bool)
- )
-
- for _, art := range artifacts {
- if values[art.ArtifactName] {
- continue
- }
- artifactHash := fmt.Sprintf("%x", md5.Sum([]byte(art.ArtifactName)))
- item := listArtifactsResponseItem{
- Name: art.ArtifactName,
- FileContainerResourceURL: ar.buildArtifactURL(runID, artifactHash, "download_url"),
- }
- items = append(items, item)
- values[art.ArtifactName] = true
-
- log.Debug("[artifact] handleListArtifacts, name: %s, url: %s", item.Name, item.FileContainerResourceURL)
- }
-
- respData := listArtifactsResponse{
- Count: int64(len(items)),
- Value: items,
- }
- ctx.JSON(http.StatusOK, respData)
- }
-
- type (
- downloadArtifactResponse struct {
- Value []downloadArtifactResponseItem `json:"value"`
- }
- downloadArtifactResponseItem struct {
- Path string `json:"path"`
- ItemType string `json:"itemType"`
- ContentLocation string `json:"contentLocation"`
- }
- )
-
- // getDownloadArtifactURL generates download url for each artifact
- func (ar artifactRoutes) getDownloadArtifactURL(ctx *ArtifactContext) {
- _, runID, ok := validateRunID(ctx)
- if !ok {
- return
- }
-
- itemPath := util.PathJoinRel(ctx.Req.URL.Query().Get("itemPath"))
- if !validateArtifactHash(ctx, itemPath) {
- return
- }
-
- artifacts, err := db.Find[actions.ActionArtifact](ctx, actions.FindArtifactsOptions{
- RunID: runID,
- ArtifactName: itemPath,
- })
- if err != nil {
- log.Error("Error getting artifacts: %v", err)
- ctx.Error(http.StatusInternalServerError, err.Error())
- return
- }
- if len(artifacts) == 0 {
- log.Debug("[artifact] getDownloadArtifactURL, no artifacts")
- ctx.Error(http.StatusNotFound)
- return
- }
-
- if itemPath != artifacts[0].ArtifactName {
- log.Error("Error dismatch artifact name, itemPath: %v, artifact: %v", itemPath, artifacts[0].ArtifactName)
- ctx.Error(http.StatusBadRequest, "Error dismatch artifact name")
- return
- }
-
- var items []downloadArtifactResponseItem
- for _, artifact := range artifacts {
- var downloadURL string
- if setting.Actions.ArtifactStorage.MinioConfig.ServeDirect {
- u, err := ar.fs.URL(artifact.StoragePath, artifact.ArtifactName)
- if err != nil && !errors.Is(err, storage.ErrURLNotSupported) {
- log.Error("Error getting serve direct url: %v", err)
- }
- if u != nil {
- downloadURL = u.String()
- }
- }
- if downloadURL == "" {
- downloadURL = ar.buildArtifactURL(runID, strconv.FormatInt(artifact.ID, 10), "download")
- }
- item := downloadArtifactResponseItem{
- Path: util.PathJoinRel(itemPath, artifact.ArtifactPath),
- ItemType: "file",
- ContentLocation: downloadURL,
- }
- log.Debug("[artifact] getDownloadArtifactURL, path: %s, url: %s", item.Path, item.ContentLocation)
- items = append(items, item)
- }
- respData := downloadArtifactResponse{
- Value: items,
- }
- ctx.JSON(http.StatusOK, respData)
- }
-
- // downloadArtifact downloads artifact content
- func (ar artifactRoutes) downloadArtifact(ctx *ArtifactContext) {
- _, runID, ok := validateRunID(ctx)
- if !ok {
- return
- }
-
- artifactID := ctx.ParamsInt64("artifact_id")
- artifact, exist, err := db.GetByID[actions.ActionArtifact](ctx, artifactID)
- if err != nil {
- log.Error("Error getting artifact: %v", err)
- ctx.Error(http.StatusInternalServerError, err.Error())
- return
- } else if !exist {
- log.Error("artifact with ID %d does not exist", artifactID)
- ctx.Error(http.StatusNotFound, fmt.Sprintf("artifact with ID %d does not exist", artifactID))
- return
- }
- if artifact.RunID != runID {
- log.Error("Error dismatch runID and artifactID, task: %v, artifact: %v", runID, artifactID)
- ctx.Error(http.StatusBadRequest, err.Error())
- return
- }
-
- fd, err := ar.fs.Open(artifact.StoragePath)
- if err != nil {
- log.Error("Error opening file: %v", err)
- ctx.Error(http.StatusInternalServerError, err.Error())
- return
- }
- defer fd.Close()
-
- // if artifact is compressed, set content-encoding header to gzip
- if artifact.ContentEncoding == "gzip" {
- ctx.Resp.Header().Set("Content-Encoding", "gzip")
- }
- log.Debug("[artifact] downloadArtifact, name: %s, path: %s, storage: %s, size: %d", artifact.ArtifactName, artifact.ArtifactPath, artifact.StoragePath, artifact.FileSize)
- ctx.ServeContent(fd, &context.ServeHeaderOptions{
- Filename: artifact.ArtifactName,
- LastModified: artifact.CreatedUnix.AsLocalTime(),
- })
- }
|