aboutsummaryrefslogtreecommitdiffstats
path: root/routers/api/packages/container
diff options
context:
space:
mode:
Diffstat (limited to 'routers/api/packages/container')
-rw-r--r--routers/api/packages/container/auth.go2
-rw-r--r--routers/api/packages/container/blob.go37
-rw-r--r--routers/api/packages/container/container.go122
-rw-r--r--routers/api/packages/container/manifest.go329
4 files changed, 221 insertions, 269 deletions
diff --git a/routers/api/packages/container/auth.go b/routers/api/packages/container/auth.go
index 1d8ae6af7d..1e1b87eb79 100644
--- a/routers/api/packages/container/auth.go
+++ b/routers/api/packages/container/auth.go
@@ -21,7 +21,7 @@ func (a *Auth) Name() string {
}
// Verify extracts the user from the Bearer token
-// If it's an anonymous session a ghost user is returned
+// If it's an anonymous session, a ghost user is returned
func (a *Auth) Verify(req *http.Request, w http.ResponseWriter, store auth.DataStore, sess auth.SessionStore) (*user_model.User, error) {
packageMeta, err := packages.ParseAuthorizationRequest(req)
if err != nil {
diff --git a/routers/api/packages/container/blob.go b/routers/api/packages/container/blob.go
index 4a2320ab76..4b7bcee9d0 100644
--- a/routers/api/packages/container/blob.go
+++ b/routers/api/packages/container/blob.go
@@ -20,11 +20,13 @@ import (
container_module "code.gitea.io/gitea/modules/packages/container"
"code.gitea.io/gitea/modules/util"
packages_service "code.gitea.io/gitea/services/packages"
+
+ "github.com/opencontainers/go-digest"
)
// saveAsPackageBlob creates a package blob from an upload
// The uploaded blob gets stored in a special upload version to link them to the package/image
-func saveAsPackageBlob(ctx context.Context, hsr packages_module.HashedSizeReader, pci *packages_service.PackageCreationInfo) (*packages_model.PackageBlob, error) { //nolint:unparam
+func saveAsPackageBlob(ctx context.Context, hsr packages_module.HashedSizeReader, pci *packages_service.PackageCreationInfo) (*packages_model.PackageBlob, error) { //nolint:unparam // PackageBlob is never used
pb := packages_service.NewPackageBlob(hsr)
exists := false
@@ -88,20 +90,18 @@ func mountBlob(ctx context.Context, pi *packages_service.PackageInfo, pb *packag
})
}
-func containerPkgName(piOwnerID int64, piName string) string {
- return fmt.Sprintf("pkg_%d_container_%s", piOwnerID, strings.ToLower(piName))
+func containerGlobalLockKey(piOwnerID int64, piName, usage string) string {
+ return fmt.Sprintf("pkg_%d_container_%s_%s", piOwnerID, strings.ToLower(piName), usage)
}
func getOrCreateUploadVersion(ctx context.Context, pi *packages_service.PackageInfo) (*packages_model.PackageVersion, error) {
- var uploadVersion *packages_model.PackageVersion
-
- releaser, err := globallock.Lock(ctx, containerPkgName(pi.Owner.ID, pi.Name))
+ releaser, err := globallock.Lock(ctx, containerGlobalLockKey(pi.Owner.ID, pi.Name, "package"))
if err != nil {
return nil, err
}
defer releaser()
- err = db.WithTx(ctx, func(ctx context.Context) error {
+ return db.WithTx2(ctx, func(ctx context.Context) (*packages_model.PackageVersion, error) {
created := true
p := &packages_model.Package{
OwnerID: pi.Owner.ID,
@@ -113,7 +113,7 @@ func getOrCreateUploadVersion(ctx context.Context, pi *packages_service.PackageI
if p, err = packages_model.TryInsertPackage(ctx, p); err != nil {
if !errors.Is(err, packages_model.ErrDuplicatePackage) {
log.Error("Error inserting package: %v", err)
- return err
+ return nil, err
}
created = false
}
@@ -121,31 +121,26 @@ func getOrCreateUploadVersion(ctx context.Context, pi *packages_service.PackageI
if created {
if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository, strings.ToLower(pi.Owner.LowerName+"/"+pi.Name)); err != nil {
log.Error("Error setting package property: %v", err)
- return err
+ return nil, err
}
}
pv := &packages_model.PackageVersion{
PackageID: p.ID,
CreatorID: pi.Owner.ID,
- Version: container_model.UploadVersion,
- LowerVersion: container_model.UploadVersion,
+ Version: container_module.UploadVersion,
+ LowerVersion: container_module.UploadVersion,
IsInternal: true,
MetadataJSON: "null",
}
if pv, err = packages_model.GetOrInsertVersion(ctx, pv); err != nil {
if !errors.Is(err, packages_model.ErrDuplicatePackageVersion) {
log.Error("Error inserting package: %v", err)
- return err
+ return nil, err
}
}
-
- uploadVersion = pv
-
- return nil
+ return pv, nil
})
-
- return uploadVersion, err
}
func createFileForBlob(ctx context.Context, pv *packages_model.PackageVersion, pb *packages_model.PackageBlob) error {
@@ -175,8 +170,8 @@ func createFileForBlob(ctx context.Context, pv *packages_model.PackageVersion, p
return nil
}
-func deleteBlob(ctx context.Context, ownerID int64, image, digest string) error {
- releaser, err := globallock.Lock(ctx, containerPkgName(ownerID, image))
+func deleteBlob(ctx context.Context, ownerID int64, image string, digest digest.Digest) error {
+ releaser, err := globallock.Lock(ctx, containerGlobalLockKey(ownerID, image, "blob"))
if err != nil {
return err
}
@@ -186,7 +181,7 @@ func deleteBlob(ctx context.Context, ownerID int64, image, digest string) error
pfds, err := container_model.GetContainerBlobs(ctx, &container_model.BlobSearchOptions{
OwnerID: ownerID,
Image: image,
- Digest: digest,
+ Digest: string(digest),
})
if err != nil {
return err
diff --git a/routers/api/packages/container/container.go b/routers/api/packages/container/container.go
index 6ef1655235..d532f698ad 100644
--- a/routers/api/packages/container/container.go
+++ b/routers/api/packages/container/container.go
@@ -13,6 +13,7 @@ import (
"regexp"
"strconv"
"strings"
+ "sync"
auth_model "code.gitea.io/gitea/models/auth"
packages_model "code.gitea.io/gitea/models/packages"
@@ -21,6 +22,7 @@ import (
"code.gitea.io/gitea/modules/httplib"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"
+ "code.gitea.io/gitea/modules/optional"
packages_module "code.gitea.io/gitea/modules/packages"
container_module "code.gitea.io/gitea/modules/packages/container"
"code.gitea.io/gitea/modules/setting"
@@ -31,17 +33,21 @@ import (
packages_service "code.gitea.io/gitea/services/packages"
container_service "code.gitea.io/gitea/services/packages/container"
- digest "github.com/opencontainers/go-digest"
+ "github.com/opencontainers/go-digest"
)
// maximum size of a container manifest
// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-manifests
const maxManifestSize = 10 * 1024 * 1024
-var (
- imageNamePattern = regexp.MustCompile(`\A[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*\z`)
- referencePattern = regexp.MustCompile(`\A[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}\z`)
-)
+var globalVars = sync.OnceValue(func() (ret struct {
+ imageNamePattern, referencePattern *regexp.Regexp
+},
+) {
+ ret.imageNamePattern = regexp.MustCompile(`\A[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*\z`)
+ ret.referencePattern = regexp.MustCompile(`\A[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}\z`)
+ return ret
+})
type containerHeaders struct {
Status int
@@ -50,7 +56,7 @@ type containerHeaders struct {
Range string
Location string
ContentType string
- ContentLength int64
+ ContentLength optional.Option[int64]
}
// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#legacy-docker-support-http-headers
@@ -64,8 +70,8 @@ func setResponseHeaders(resp http.ResponseWriter, h *containerHeaders) {
if h.ContentType != "" {
resp.Header().Set("Content-Type", h.ContentType)
}
- if h.ContentLength != 0 {
- resp.Header().Set("Content-Length", strconv.FormatInt(h.ContentLength, 10))
+ if h.ContentLength.Has() {
+ resp.Header().Set("Content-Length", strconv.FormatInt(h.ContentLength.Value(), 10))
}
if h.UploadUUID != "" {
resp.Header().Set("Docker-Upload-Uuid", h.UploadUUID)
@@ -83,9 +89,7 @@ func jsonResponse(ctx *context.Context, status int, obj any) {
Status: status,
ContentType: "application/json",
})
- if err := json.NewEncoder(ctx.Resp).Encode(obj); err != nil {
- log.Error("JSON encode: %v", err)
- }
+ _ = json.NewEncoder(ctx.Resp).Encode(obj) // ignore network errors
}
func apiError(ctx *context.Context, status int, err error) {
@@ -133,7 +137,7 @@ func ReqContainerAccess(ctx *context.Context) {
// VerifyImageName is a middleware which checks if the image name is allowed
func VerifyImageName(ctx *context.Context) {
- if !imageNamePattern.MatchString(ctx.PathParam("image")) {
+ if !globalVars().imageNamePattern.MatchString(ctx.PathParam("image")) {
apiErrorDefined(ctx, errNameInvalid)
}
}
@@ -215,7 +219,7 @@ func GetRepositoryList(ctx *context.Context) {
if len(repositories) == n {
v := url.Values{}
if n > 0 {
- v.Add("n", strconv.Itoa(n))
+ v.Add("n", strconv.Itoa(n)) // FIXME: "n" can't be zero here, the logic is inconsistent with GetTagsList
}
v.Add("last", repositories[len(repositories)-1])
@@ -230,7 +234,7 @@ func GetRepositoryList(ctx *context.Context) {
// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#mounting-a-blob-from-another-repository
// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#single-post
// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-a-blob-in-chunks
-func InitiateUploadBlob(ctx *context.Context) {
+func PostBlobsUploads(ctx *context.Context) {
image := ctx.PathParam("image")
mount := ctx.FormTrim("mount")
@@ -312,14 +316,14 @@ func InitiateUploadBlob(ctx *context.Context) {
setResponseHeaders(ctx.Resp, &containerHeaders{
Location: fmt.Sprintf("/v2/%s/%s/blobs/uploads/%s", ctx.Package.Owner.LowerName, image, upload.ID),
- Range: "0-0",
UploadUUID: upload.ID,
Status: http.StatusAccepted,
})
}
-// https://docs.docker.com/registry/spec/api/#get-blob-upload
-func GetUploadBlob(ctx *context.Context) {
+// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-a-blob-in-chunks
+func GetBlobsUpload(ctx *context.Context) {
+ image := ctx.PathParam("image")
uuid := ctx.PathParam("uuid")
upload, err := packages_model.GetBlobUploadByID(ctx, uuid)
@@ -332,15 +336,21 @@ func GetUploadBlob(ctx *context.Context) {
return
}
- setResponseHeaders(ctx.Resp, &containerHeaders{
- Range: fmt.Sprintf("0-%d", upload.BytesReceived),
+ // FIXME: undefined behavior when the uploaded content is empty: https://github.com/opencontainers/distribution-spec/issues/578
+ respHeaders := &containerHeaders{
+ Location: fmt.Sprintf("/v2/%s/%s/blobs/uploads/%s", ctx.Package.Owner.LowerName, image, upload.ID),
UploadUUID: upload.ID,
Status: http.StatusNoContent,
- })
+ }
+ if upload.BytesReceived > 0 {
+ respHeaders.Range = fmt.Sprintf("0-%d", upload.BytesReceived-1)
+ }
+ setResponseHeaders(ctx.Resp, respHeaders)
}
+// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#single-post
// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-a-blob-in-chunks
-func UploadBlob(ctx *context.Context) {
+func PatchBlobsUpload(ctx *context.Context) {
image := ctx.PathParam("image")
uploader, err := container_service.NewBlobUploader(ctx, ctx.PathParam("uuid"))
@@ -376,16 +386,19 @@ func UploadBlob(ctx *context.Context) {
return
}
- setResponseHeaders(ctx.Resp, &containerHeaders{
+ respHeaders := &containerHeaders{
Location: fmt.Sprintf("/v2/%s/%s/blobs/uploads/%s", ctx.Package.Owner.LowerName, image, uploader.ID),
- Range: fmt.Sprintf("0-%d", uploader.Size()-1),
UploadUUID: uploader.ID,
Status: http.StatusAccepted,
- })
+ }
+ if uploader.Size() > 0 {
+ respHeaders.Range = fmt.Sprintf("0-%d", uploader.Size()-1)
+ }
+ setResponseHeaders(ctx.Resp, respHeaders)
}
// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-a-blob-in-chunks
-func EndUploadBlob(ctx *context.Context) {
+func PutBlobsUpload(ctx *context.Context) {
image := ctx.PathParam("image")
digest := ctx.FormTrim("digest")
@@ -403,12 +416,7 @@ func EndUploadBlob(ctx *context.Context) {
}
return
}
- doClose := true
- defer func() {
- if doClose {
- uploader.Close()
- }
- }()
+ defer uploader.Close()
if ctx.Req.Body != nil {
if err := uploader.Append(ctx, ctx.Req.Body); err != nil {
@@ -441,11 +449,10 @@ func EndUploadBlob(ctx *context.Context) {
return
}
- if err := uploader.Close(); err != nil {
- apiError(ctx, http.StatusInternalServerError, err)
- return
- }
- doClose = false
+ // There was a strange bug: the "Close" fails with error "close .../tmp/package-upload/....: file already closed"
+ // AFAIK there should be no other "Close" call to the uploader between NewBlobUploader and this line.
+ // At least it's safe to call Close twice, so ignore the error.
+ _ = uploader.Close()
if err := container_service.RemoveBlobUploadByID(ctx, uploader.ID); err != nil {
apiError(ctx, http.StatusInternalServerError, err)
@@ -460,7 +467,7 @@ func EndUploadBlob(ctx *context.Context) {
}
// https://docs.docker.com/registry/spec/api/#delete-blob-upload
-func CancelUploadBlob(ctx *context.Context) {
+func DeleteBlobsUpload(ctx *context.Context) {
uuid := ctx.PathParam("uuid")
_, err := packages_model.GetBlobUploadByID(ctx, uuid)
@@ -484,16 +491,15 @@ func CancelUploadBlob(ctx *context.Context) {
}
func getBlobFromContext(ctx *context.Context) (*packages_model.PackageFileDescriptor, error) {
- d := ctx.PathParam("digest")
-
- if digest.Digest(d).Validate() != nil {
+ d := digest.Digest(ctx.PathParam("digest"))
+ if d.Validate() != nil {
return nil, container_model.ErrContainerBlobNotExist
}
return workaroundGetContainerBlob(ctx, &container_model.BlobSearchOptions{
OwnerID: ctx.Package.Owner.ID,
Image: ctx.PathParam("image"),
- Digest: d,
+ Digest: string(d),
})
}
@@ -511,7 +517,7 @@ func HeadBlob(ctx *context.Context) {
setResponseHeaders(ctx.Resp, &containerHeaders{
ContentDigest: blob.Properties.GetByName(container_module.PropertyDigest),
- ContentLength: blob.Blob.Size,
+ ContentLength: optional.Some(blob.Blob.Size),
Status: http.StatusOK,
})
}
@@ -533,9 +539,8 @@ func GetBlob(ctx *context.Context) {
// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#deleting-blobs
func DeleteBlob(ctx *context.Context) {
- d := ctx.PathParam("digest")
-
- if digest.Digest(d).Validate() != nil {
+ d := digest.Digest(ctx.PathParam("digest"))
+ if d.Validate() != nil {
apiErrorDefined(ctx, errBlobUnknown)
return
}
@@ -551,7 +556,7 @@ func DeleteBlob(ctx *context.Context) {
}
// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-manifests
-func UploadManifest(ctx *context.Context) {
+func PutManifest(ctx *context.Context) {
reference := ctx.PathParam("reference")
mci := &manifestCreationInfo{
@@ -563,7 +568,7 @@ func UploadManifest(ctx *context.Context) {
IsTagged: digest.Digest(reference).Validate() != nil,
}
- if mci.IsTagged && !referencePattern.MatchString(reference) {
+ if mci.IsTagged && !globalVars().referencePattern.MatchString(reference) {
apiErrorDefined(ctx, errManifestInvalid.WithMessage("Tag is invalid"))
return
}
@@ -607,18 +612,18 @@ func UploadManifest(ctx *context.Context) {
}
func getBlobSearchOptionsFromContext(ctx *context.Context) (*container_model.BlobSearchOptions, error) {
- reference := ctx.PathParam("reference")
-
opts := &container_model.BlobSearchOptions{
OwnerID: ctx.Package.Owner.ID,
Image: ctx.PathParam("image"),
IsManifest: true,
}
- if digest.Digest(reference).Validate() == nil {
- opts.Digest = reference
- } else if referencePattern.MatchString(reference) {
+ reference := ctx.PathParam("reference")
+ if d := digest.Digest(reference); d.Validate() == nil {
+ opts.Digest = string(d)
+ } else if globalVars().referencePattern.MatchString(reference) {
opts.Tag = reference
+ opts.OnlyLead = true
} else {
return nil, container_model.ErrContainerBlobNotExist
}
@@ -650,7 +655,7 @@ func HeadManifest(ctx *context.Context) {
setResponseHeaders(ctx.Resp, &containerHeaders{
ContentDigest: manifest.Properties.GetByName(container_module.PropertyDigest),
ContentType: manifest.Properties.GetByName(container_module.PropertyMediaType),
- ContentLength: manifest.Blob.Size,
+ ContentLength: optional.Some(manifest.Blob.Size),
Status: http.StatusOK,
})
}
@@ -705,7 +710,7 @@ func DeleteManifest(ctx *context.Context) {
func serveBlob(ctx *context.Context, pfd *packages_model.PackageFileDescriptor) {
serveDirectReqParams := make(url.Values)
serveDirectReqParams.Set("response-content-type", pfd.Properties.GetByName(container_module.PropertyMediaType))
- s, u, _, err := packages_service.GetPackageBlobStream(ctx, pfd.File, pfd.Blob, serveDirectReqParams)
+ s, u, _, err := packages_service.OpenBlobForDownload(ctx, pfd.File, pfd.Blob, serveDirectReqParams)
if err != nil {
apiError(ctx, http.StatusInternalServerError, err)
return
@@ -714,14 +719,14 @@ func serveBlob(ctx *context.Context, pfd *packages_model.PackageFileDescriptor)
headers := &containerHeaders{
ContentDigest: pfd.Properties.GetByName(container_module.PropertyDigest),
ContentType: pfd.Properties.GetByName(container_module.PropertyMediaType),
- ContentLength: pfd.Blob.Size,
+ ContentLength: optional.Some(pfd.Blob.Size),
Status: http.StatusOK,
}
if u != nil {
headers.Status = http.StatusTemporaryRedirect
headers.Location = u.String()
-
+ headers.ContentLength = optional.None[int64]() // do not set Content-Length for redirect responses
setResponseHeaders(ctx.Resp, headers)
return
}
@@ -735,7 +740,7 @@ func serveBlob(ctx *context.Context, pfd *packages_model.PackageFileDescriptor)
}
// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#content-discovery
-func GetTagList(ctx *context.Context) {
+func GetTagsList(ctx *context.Context) {
image := ctx.PathParam("image")
if _, err := packages_model.GetPackageByName(ctx, ctx.Package.Owner.ID, packages_model.TypeContainer, image); err != nil {
@@ -780,7 +785,8 @@ func GetTagList(ctx *context.Context) {
})
}
-// FIXME: Workaround to be removed in v1.20
+// FIXME: Workaround to be removed in v1.20.
+// Update maybe we should never really remote it, as long as there is legacy data?
// https://github.com/go-gitea/gitea/issues/19586
func workaroundGetContainerBlob(ctx *context.Context, opts *container_model.BlobSearchOptions) (*packages_model.PackageFileDescriptor, error) {
blob, err := container_model.GetContainerBlob(ctx, opts)
diff --git a/routers/api/packages/container/manifest.go b/routers/api/packages/container/manifest.go
index 26faa7b024..de40215aa7 100644
--- a/routers/api/packages/container/manifest.go
+++ b/routers/api/packages/container/manifest.go
@@ -10,11 +10,13 @@ import (
"io"
"os"
"strings"
+ "time"
"code.gitea.io/gitea/models/db"
packages_model "code.gitea.io/gitea/models/packages"
container_model "code.gitea.io/gitea/models/packages/container"
user_model "code.gitea.io/gitea/models/user"
+ "code.gitea.io/gitea/modules/globallock"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"
packages_module "code.gitea.io/gitea/modules/packages"
@@ -22,23 +24,12 @@ import (
"code.gitea.io/gitea/modules/util"
notify_service "code.gitea.io/gitea/services/notify"
packages_service "code.gitea.io/gitea/services/packages"
+ container_service "code.gitea.io/gitea/services/packages/container"
- digest "github.com/opencontainers/go-digest"
+ "github.com/opencontainers/go-digest"
oci "github.com/opencontainers/image-spec/specs-go/v1"
)
-func isValidMediaType(mt string) bool {
- return strings.HasPrefix(mt, "application/vnd.docker.") || strings.HasPrefix(mt, "application/vnd.oci.")
-}
-
-func isImageManifestMediaType(mt string) bool {
- return strings.EqualFold(mt, oci.MediaTypeImageManifest) || strings.EqualFold(mt, "application/vnd.docker.distribution.manifest.v2+json")
-}
-
-func isImageIndexMediaType(mt string) bool {
- return strings.EqualFold(mt, oci.MediaTypeImageIndex) || strings.EqualFold(mt, "application/vnd.docker.distribution.manifest.list.v2+json")
-}
-
// manifestCreationInfo describes a manifest to create
type manifestCreationInfo struct {
MediaType string
@@ -55,71 +46,71 @@ func processManifest(ctx context.Context, mci *manifestCreationInfo, buf *packag
if err := json.NewDecoder(buf).Decode(&index); err != nil {
return "", err
}
-
if index.SchemaVersion != 2 {
return "", errUnsupported.WithMessage("Schema version is not supported")
}
-
if _, err := buf.Seek(0, io.SeekStart); err != nil {
return "", err
}
- if !isValidMediaType(mci.MediaType) {
+ if !container_module.IsMediaTypeValid(mci.MediaType) {
mci.MediaType = index.MediaType
- if !isValidMediaType(mci.MediaType) {
+ if !container_module.IsMediaTypeValid(mci.MediaType) {
return "", errManifestInvalid.WithMessage("MediaType not recognized")
}
}
- if isImageManifestMediaType(mci.MediaType) {
- return processImageManifest(ctx, mci, buf)
- } else if isImageIndexMediaType(mci.MediaType) {
- return processImageManifestIndex(ctx, mci, buf)
+ // .../container/manifest.go:453:createManifestBlob() [E] Error inserting package blob: Error 1062 (23000): Duplicate entry '..........' for key 'package_blob.UQE_package_blob_md5'
+ releaser, err := globallock.Lock(ctx, containerGlobalLockKey(mci.Owner.ID, mci.Image, "manifest"))
+ if err != nil {
+ return "", err
+ }
+ defer releaser()
+
+ if container_module.IsMediaTypeImageManifest(mci.MediaType) {
+ return processOciImageManifest(ctx, mci, buf)
+ } else if container_module.IsMediaTypeImageIndex(mci.MediaType) {
+ return processOciImageIndex(ctx, mci, buf)
}
return "", errManifestInvalid
}
-func processImageManifest(ctx context.Context, mci *manifestCreationInfo, buf *packages_module.HashedBuffer) (string, error) {
- manifestDigest := ""
-
- err := func() error {
- var manifest oci.Manifest
- if err := json.NewDecoder(buf).Decode(&manifest); err != nil {
- return err
- }
-
- if _, err := buf.Seek(0, io.SeekStart); err != nil {
- return err
- }
-
- ctx, committer, err := db.TxContext(ctx)
- if err != nil {
- return err
- }
- defer committer.Close()
-
- configDescriptor, err := container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{
- OwnerID: mci.Owner.ID,
- Image: mci.Image,
- Digest: string(manifest.Config.Digest),
- })
- if err != nil {
- return err
- }
+type processManifestTxRet struct {
+ pv *packages_model.PackageVersion
+ pb *packages_model.PackageBlob
+ created bool
+ digest string
+}
- configReader, err := packages_module.NewContentStore().Get(packages_module.BlobHash256Key(configDescriptor.Blob.HashSHA256))
- if err != nil {
- return err
+func handleCreateManifestResult(ctx context.Context, err error, mci *manifestCreationInfo, contentStore *packages_module.ContentStore, txRet *processManifestTxRet) (string, error) {
+ if err != nil && txRet.created && txRet.pb != nil {
+ if err := contentStore.Delete(packages_module.BlobHash256Key(txRet.pb.HashSHA256)); err != nil {
+ log.Error("Error deleting package blob from content store: %v", err)
}
- defer configReader.Close()
+ return "", err
+ }
+ pd, err := packages_model.GetPackageDescriptor(ctx, txRet.pv)
+ if err != nil {
+ log.Error("Error getting package descriptor: %v", err) // ignore this error
+ } else {
+ notify_service.PackageCreate(ctx, mci.Creator, pd)
+ }
+ return txRet.digest, nil
+}
- metadata, err := container_module.ParseImageConfig(manifest.Config.MediaType, configReader)
- if err != nil {
- return err
- }
+func processOciImageManifest(ctx context.Context, mci *manifestCreationInfo, buf *packages_module.HashedBuffer) (manifestDigest string, errRet error) {
+ manifest, configDescriptor, metadata, err := container_service.ParseManifestMetadata(ctx, buf, mci.Owner.ID, mci.Image)
+ if err != nil {
+ return "", err
+ }
+ if _, err = buf.Seek(0, io.SeekStart); err != nil {
+ return "", err
+ }
+ contentStore := packages_module.NewContentStore()
+ var txRet processManifestTxRet
+ err = db.WithTx(ctx, func(ctx context.Context) (err error) {
blobReferences := make([]*blobReference, 0, 1+len(manifest.Layers))
-
blobReferences = append(blobReferences, &blobReference{
Digest: manifest.Config.Digest,
MediaType: manifest.Config.MediaType,
@@ -150,78 +141,43 @@ func processImageManifest(ctx context.Context, mci *manifestCreationInfo, buf *p
return err
}
- uploadVersion, err := packages_model.GetInternalVersionByNameAndVersion(ctx, mci.Owner.ID, packages_model.TypeContainer, mci.Image, container_model.UploadVersion)
- if err != nil && err != packages_model.ErrPackageNotExist {
+ uploadVersion, err := packages_model.GetInternalVersionByNameAndVersion(ctx, mci.Owner.ID, packages_model.TypeContainer, mci.Image, container_module.UploadVersion)
+ if err != nil && !errors.Is(err, packages_model.ErrPackageNotExist) {
return err
}
for _, ref := range blobReferences {
- if err := createFileFromBlobReference(ctx, pv, uploadVersion, ref); err != nil {
+ if _, err = createFileFromBlobReference(ctx, pv, uploadVersion, ref); err != nil {
return err
}
}
+ txRet.pv = pv
+ txRet.pb, txRet.created, txRet.digest, err = createManifestBlob(ctx, contentStore, mci, pv, buf)
+ return err
+ })
- pb, created, digest, err := createManifestBlob(ctx, mci, pv, buf)
- removeBlob := false
- defer func() {
- if removeBlob {
- contentStore := packages_module.NewContentStore()
- if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
- log.Error("Error deleting package blob from content store: %v", err)
- }
- }
- }()
- if err != nil {
- removeBlob = created
- return err
- }
-
- if err := committer.Commit(); err != nil {
- removeBlob = created
- return err
- }
-
- if err := notifyPackageCreate(ctx, mci.Creator, pv); err != nil {
- return err
- }
-
- manifestDigest = digest
+ return handleCreateManifestResult(ctx, err, mci, contentStore, &txRet)
+}
- return nil
- }()
- if err != nil {
+func processOciImageIndex(ctx context.Context, mci *manifestCreationInfo, buf *packages_module.HashedBuffer) (manifestDigest string, errRet error) {
+ var index oci.Index
+ if err := json.NewDecoder(buf).Decode(&index); err != nil {
+ return "", err
+ }
+ if _, err := buf.Seek(0, io.SeekStart); err != nil {
return "", err
}
- return manifestDigest, nil
-}
-
-func processImageManifestIndex(ctx context.Context, mci *manifestCreationInfo, buf *packages_module.HashedBuffer) (string, error) {
- manifestDigest := ""
-
- err := func() error {
- var index oci.Index
- if err := json.NewDecoder(buf).Decode(&index); err != nil {
- return err
- }
-
- if _, err := buf.Seek(0, io.SeekStart); err != nil {
- return err
- }
-
- ctx, committer, err := db.TxContext(ctx)
- if err != nil {
- return err
- }
- defer committer.Close()
-
+ contentStore := packages_module.NewContentStore()
+ var txRet processManifestTxRet
+ err := db.WithTx(ctx, func(ctx context.Context) (err error) {
metadata := &container_module.Metadata{
Type: container_module.TypeOCI,
Manifests: make([]*container_module.Manifest, 0, len(index.Manifests)),
}
for _, manifest := range index.Manifests {
- if !isImageManifestMediaType(manifest.MediaType) {
+ if !container_module.IsMediaTypeImageManifest(manifest.MediaType) {
return errManifestInvalid
}
@@ -265,50 +221,12 @@ func processImageManifestIndex(ctx context.Context, mci *manifestCreationInfo, b
return err
}
- pb, created, digest, err := createManifestBlob(ctx, mci, pv, buf)
- removeBlob := false
- defer func() {
- if removeBlob {
- contentStore := packages_module.NewContentStore()
- if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
- log.Error("Error deleting package blob from content store: %v", err)
- }
- }
- }()
- if err != nil {
- removeBlob = created
- return err
- }
-
- if err := committer.Commit(); err != nil {
- removeBlob = created
- return err
- }
-
- if err := notifyPackageCreate(ctx, mci.Creator, pv); err != nil {
- return err
- }
-
- manifestDigest = digest
-
- return nil
- }()
- if err != nil {
- return "", err
- }
-
- return manifestDigest, nil
-}
-
-func notifyPackageCreate(ctx context.Context, doer *user_model.User, pv *packages_model.PackageVersion) error {
- pd, err := packages_model.GetPackageDescriptor(ctx, pv)
- if err != nil {
+ txRet.pv = pv
+ txRet.pb, txRet.created, txRet.digest, err = createManifestBlob(ctx, contentStore, mci, pv, buf)
return err
- }
-
- notify_service.PackageCreate(ctx, doer, pd)
+ })
- return nil
+ return handleCreateManifestResult(ctx, err, mci, contentStore, &txRet)
}
func createPackageAndVersion(ctx context.Context, mci *manifestCreationInfo, metadata *container_module.Metadata) (*packages_model.PackageVersion, error) {
@@ -349,24 +267,31 @@ func createPackageAndVersion(ctx context.Context, mci *manifestCreationInfo, met
LowerVersion: strings.ToLower(mci.Reference),
MetadataJSON: string(metadataJSON),
}
- var pv *packages_model.PackageVersion
- if pv, err = packages_model.GetOrInsertVersion(ctx, _pv); err != nil {
+ pv, err := packages_model.GetOrInsertVersion(ctx, _pv)
+ if err != nil {
if !errors.Is(err, packages_model.ErrDuplicatePackageVersion) {
log.Error("Error inserting package: %v", err)
return nil, err
}
- if err = packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil {
- return nil, err
- }
-
- // keep download count on overwrite
- _pv.DownloadCount = pv.DownloadCount
-
- if pv, err = packages_model.GetOrInsertVersion(ctx, _pv); err != nil {
- if !errors.Is(err, packages_model.ErrDuplicatePackageVersion) {
- log.Error("Error inserting package: %v", err)
- return nil, err
+ if container_module.IsMediaTypeImageIndex(mci.MediaType) {
+ if pv.CreatedUnix.AsTime().Before(time.Now().Add(-24 * time.Hour)) {
+ if err = packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil {
+ return nil, err
+ }
+ // keep download count on overwriting
+ _pv.DownloadCount = pv.DownloadCount
+ if pv, err = packages_model.GetOrInsertVersion(ctx, _pv); err != nil {
+ if !errors.Is(err, packages_model.ErrDuplicatePackageVersion) {
+ log.Error("Error inserting package: %v", err)
+ return nil, err
+ }
+ }
+ } else {
+ err = packages_model.UpdateVersion(ctx, &packages_model.PackageVersion{ID: pv.ID, MetadataJSON: _pv.MetadataJSON})
+ if err != nil {
+ return nil, err
+ }
}
}
}
@@ -376,14 +301,20 @@ func createPackageAndVersion(ctx context.Context, mci *manifestCreationInfo, met
}
if mci.IsTagged {
- if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestTagged, ""); err != nil {
- log.Error("Error setting package version property: %v", err)
+ if err = packages_model.InsertOrUpdateProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestTagged, ""); err != nil {
+ return nil, err
+ }
+ } else {
+ if err = packages_model.DeletePropertiesByName(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestTagged); err != nil {
return nil, err
}
}
+
+ if err = packages_model.DeletePropertiesByName(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestReference); err != nil {
+ return nil, err
+ }
for _, manifest := range metadata.Manifests {
- if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestReference, manifest.Digest); err != nil {
- log.Error("Error setting package version property: %v", err)
+ if _, err = packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestReference, manifest.Digest); err != nil {
return nil, err
}
}
@@ -400,9 +331,9 @@ type blobReference struct {
IsLead bool
}
-func createFileFromBlobReference(ctx context.Context, pv, uploadVersion *packages_model.PackageVersion, ref *blobReference) error {
+func createFileFromBlobReference(ctx context.Context, pv, uploadVersion *packages_model.PackageVersion, ref *blobReference) (*packages_model.PackageFile, error) {
if ref.File.Blob.Size != ref.ExpectedSize {
- return errSizeInvalid
+ return nil, errSizeInvalid
}
if ref.Name == "" {
@@ -410,20 +341,21 @@ func createFileFromBlobReference(ctx context.Context, pv, uploadVersion *package
}
pf := &packages_model.PackageFile{
- VersionID: pv.ID,
- BlobID: ref.File.Blob.ID,
- Name: ref.Name,
- LowerName: ref.Name,
- IsLead: ref.IsLead,
+ VersionID: pv.ID,
+ BlobID: ref.File.Blob.ID,
+ Name: ref.Name,
+ LowerName: ref.Name,
+ CompositeKey: string(ref.Digest),
+ IsLead: ref.IsLead,
}
var err error
if pf, err = packages_model.TryInsertFile(ctx, pf); err != nil {
if errors.Is(err, packages_model.ErrDuplicatePackageFile) {
// Skip this blob because the manifest contains the same filesystem layer multiple times.
- return nil
+ return pf, nil
}
log.Error("Error inserting package file: %v", err)
- return err
+ return nil, err
}
props := map[string]string{
@@ -433,21 +365,21 @@ func createFileFromBlobReference(ctx context.Context, pv, uploadVersion *package
for name, value := range props {
if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeFile, pf.ID, name, value); err != nil {
log.Error("Error setting package file property: %v", err)
- return err
+ return nil, err
}
}
- // Remove the file from the blob upload version
+ // Remove the ref file (old file) from the blob upload version
if uploadVersion != nil && ref.File.File != nil && uploadVersion.ID == ref.File.File.VersionID {
if err := packages_service.DeletePackageFile(ctx, ref.File.File); err != nil {
- return err
+ return nil, err
}
}
- return nil
+ return pf, nil
}
-func createManifestBlob(ctx context.Context, mci *manifestCreationInfo, pv *packages_model.PackageVersion, buf *packages_module.HashedBuffer) (*packages_model.PackageBlob, bool, string, error) {
+func createManifestBlob(ctx context.Context, contentStore *packages_module.ContentStore, mci *manifestCreationInfo, pv *packages_model.PackageVersion, buf *packages_module.HashedBuffer) (_ *packages_model.PackageBlob, created bool, manifestDigest string, _ error) {
pb, exists, err := packages_model.GetOrInsertBlob(ctx, packages_service.NewPackageBlob(buf))
if err != nil {
log.Error("Error inserting package blob: %v", err)
@@ -456,29 +388,48 @@ func createManifestBlob(ctx context.Context, mci *manifestCreationInfo, pv *pack
// FIXME: Workaround to be removed in v1.20
// https://github.com/go-gitea/gitea/issues/19586
if exists {
- err = packages_module.NewContentStore().Has(packages_module.BlobHash256Key(pb.HashSHA256))
+ err = contentStore.Has(packages_module.BlobHash256Key(pb.HashSHA256))
if err != nil && (errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist)) {
log.Debug("Package registry inconsistent: blob %s does not exist on file system", pb.HashSHA256)
exists = false
}
}
if !exists {
- contentStore := packages_module.NewContentStore()
if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), buf, buf.Size()); err != nil {
log.Error("Error saving package blob in content store: %v", err)
return nil, false, "", err
}
}
- manifestDigest := digestFromHashSummer(buf)
- err = createFileFromBlobReference(ctx, pv, nil, &blobReference{
+ manifestDigest = digestFromHashSummer(buf)
+ pf, err := createFileFromBlobReference(ctx, pv, nil, &blobReference{
Digest: digest.Digest(manifestDigest),
MediaType: mci.MediaType,
- Name: container_model.ManifestFilename,
+ Name: container_module.ManifestFilename,
File: &packages_model.PackageFileDescriptor{Blob: pb},
ExpectedSize: pb.Size,
IsLead: true,
})
+ if err != nil {
+ return nil, false, "", err
+ }
+ oldManifestFiles, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{
+ OwnerID: mci.Owner.ID,
+ PackageType: packages_model.TypeContainer,
+ VersionID: pv.ID,
+ Query: container_module.ManifestFilename,
+ })
+ if err != nil {
+ return nil, false, "", err
+ }
+ for _, oldManifestFile := range oldManifestFiles {
+ if oldManifestFile.ID != pf.ID && oldManifestFile.IsLead {
+ err = packages_model.UpdateFile(ctx, &packages_model.PackageFile{ID: oldManifestFile.ID, IsLead: false}, []string{"is_lead"})
+ if err != nil {
+ return nil, false, "", err
+ }
+ }
+ }
return pb, !exists, manifestDigest, err
}
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-01 06:25:35 +0000