diff options
Diffstat (limited to 'routers/api/packages')
28 files changed, 430 insertions, 529 deletions
diff --git a/routers/api/packages/alpine/alpine.go b/routers/api/packages/alpine/alpine.go index f35cff3df2..ba4a4f23ce 100644 --- a/routers/api/packages/alpine/alpine.go +++ b/routers/api/packages/alpine/alpine.go @@ -68,7 +68,7 @@ func GetRepositoryFile(ctx *context.Context) { return } - s, u, pf, err := packages_service.GetFileStreamByPackageVersion( + s, u, pf, err := packages_service.OpenFileForDownloadByPackageVersion( ctx, pv, &packages_service.PackageFileInfo{ @@ -216,7 +216,7 @@ func DownloadPackageFile(ctx *context.Context) { } } - s, u, pf, err := packages_service.GetPackageFileStream(ctx, pfs[0]) + s, u, pf, err := packages_service.OpenFileForDownload(ctx, pfs[0]) if err != nil { if errors.Is(err, util.ErrNotExist) { apiError(ctx, http.StatusNotFound, err) diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index ae4ea7ea87..878e0f9945 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -5,8 +5,6 @@ package packages import ( "net/http" - "regexp" - "strings" auth_model "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/perm" @@ -282,42 +280,10 @@ func CommonRoutes() *web.Router { }) }) }, reqPackageAccess(perm.AccessModeRead)) - r.Group("/conda", func() { - var ( - downloadPattern = regexp.MustCompile(`\A(.+/)?(.+)/((?:[^/]+(?:\.tar\.bz2|\.conda))|(?:current_)?repodata\.json(?:\.bz2)?)\z`) - uploadPattern = regexp.MustCompile(`\A(.+/)?([^/]+(?:\.tar\.bz2|\.conda))\z`) - ) - - r.Get("/*", func(ctx *context.Context) { - m := downloadPattern.FindStringSubmatch(ctx.PathParam("*")) - if len(m) == 0 { - ctx.Status(http.StatusNotFound) - return - } - - ctx.SetPathParam("channel", strings.TrimSuffix(m[1], "/")) - ctx.SetPathParam("architecture", m[2]) - ctx.SetPathParam("filename", m[3]) - - switch m[3] { - case "repodata.json", "repodata.json.bz2", "current_repodata.json", "current_repodata.json.bz2": - conda.EnumeratePackages(ctx) - default: - conda.DownloadPackageFile(ctx) - } - }) - r.Put("/*", reqPackageAccess(perm.AccessModeWrite), func(ctx *context.Context) { - m := uploadPattern.FindStringSubmatch(ctx.PathParam("*")) - if len(m) == 0 { - ctx.Status(http.StatusNotFound) - return - } - - ctx.SetPathParam("channel", strings.TrimSuffix(m[1], "/")) - ctx.SetPathParam("filename", m[2]) - - conda.UploadPackageFile(ctx) - }) + r.PathGroup("/conda/*", func(g *web.RouterPathGroup) { + g.MatchPath("GET", "/<architecture>/<filename>", conda.ListOrGetPackages) + g.MatchPath("GET", "/<channel:*>/<architecture>/<filename>", conda.ListOrGetPackages) + g.MatchPath("PUT", "/<channel:*>/<filename>", reqPackageAccess(perm.AccessModeWrite), conda.UploadPackageFile) }, reqPackageAccess(perm.AccessModeRead)) r.Group("/cran", func() { r.Group("/src", func() { @@ -358,60 +324,15 @@ func CommonRoutes() *web.Router { }, reqPackageAccess(perm.AccessModeRead)) r.Group("/go", func() { r.Put("/upload", reqPackageAccess(perm.AccessModeWrite), goproxy.UploadPackage) - r.Get("/sumdb/sum.golang.org/supported", func(ctx *context.Context) { - ctx.Status(http.StatusNotFound) - }) + r.Get("/sumdb/sum.golang.org/supported", http.NotFound) - // Manual mapping of routes because the package name contains slashes which chi does not support // https://go.dev/ref/mod#goproxy-protocol - r.Get("/*", func(ctx *context.Context) { - path := ctx.PathParam("*") - - if strings.HasSuffix(path, "/@latest") { - ctx.SetPathParam("name", path[:len(path)-len("/@latest")]) - ctx.SetPathParam("version", "latest") - - goproxy.PackageVersionMetadata(ctx) - return - } - - parts := strings.SplitN(path, "/@v/", 2) - if len(parts) != 2 { - ctx.Status(http.StatusNotFound) - return - } - - ctx.SetPathParam("name", parts[0]) - - // <package/name>/@v/list - if parts[1] == "list" { - goproxy.EnumeratePackageVersions(ctx) - return - } - - // <package/name>/@v/<version>.zip - if strings.HasSuffix(parts[1], ".zip") { - ctx.SetPathParam("version", parts[1][:len(parts[1])-len(".zip")]) - - goproxy.DownloadPackageFile(ctx) - return - } - // <package/name>/@v/<version>.info - if strings.HasSuffix(parts[1], ".info") { - ctx.SetPathParam("version", parts[1][:len(parts[1])-len(".info")]) - - goproxy.PackageVersionMetadata(ctx) - return - } - // <package/name>/@v/<version>.mod - if strings.HasSuffix(parts[1], ".mod") { - ctx.SetPathParam("version", parts[1][:len(parts[1])-len(".mod")]) - - goproxy.PackageVersionGoModContent(ctx) - return - } - - ctx.Status(http.StatusNotFound) + r.PathGroup("/*", func(g *web.RouterPathGroup) { + g.MatchPath("GET", "/<name:*>/@<version:latest>", goproxy.PackageVersionMetadata) + g.MatchPath("GET", "/<name:*>/@v/list", goproxy.EnumeratePackageVersions) + g.MatchPath("GET", "/<name:*>/@v/<version>.zip", goproxy.DownloadPackageFile) + g.MatchPath("GET", "/<name:*>/@v/<version>.info", goproxy.PackageVersionMetadata) + g.MatchPath("GET", "/<name:*>/@v/<version>.mod", goproxy.PackageVersionGoModContent) }) }, reqPackageAccess(perm.AccessModeRead)) r.Group("/generic", func() { @@ -532,82 +453,26 @@ func CommonRoutes() *web.Router { }) }) }, reqPackageAccess(perm.AccessModeRead)) + r.Group("/pypi", func() { r.Post("/", reqPackageAccess(perm.AccessModeWrite), pypi.UploadPackageFile) r.Get("/files/{id}/{version}/{filename}", pypi.DownloadPackageFile) r.Get("/simple/{id}", pypi.PackageMetadata) }, reqPackageAccess(perm.AccessModeRead)) - r.Group("/rpm", func() { - r.Group("/repository.key", func() { - r.Head("", rpm.GetRepositoryKey) - r.Get("", rpm.GetRepositoryKey) - }) - - var ( - repoPattern = regexp.MustCompile(`\A(.*?)\.repo\z`) - uploadPattern = regexp.MustCompile(`\A(.*?)/upload\z`) - filePattern = regexp.MustCompile(`\A(.*?)/package/([^/]+)/([^/]+)/([^/]+)(?:/([^/]+\.rpm)|)\z`) - repoFilePattern = regexp.MustCompile(`\A(.*?)/repodata/([^/]+)\z`) - ) - - r.Methods("HEAD,GET,PUT,DELETE", "*", func(ctx *context.Context) { - path := ctx.PathParam("*") - isHead := ctx.Req.Method == http.MethodHead - isGetHead := ctx.Req.Method == http.MethodHead || ctx.Req.Method == http.MethodGet - isPut := ctx.Req.Method == http.MethodPut - isDelete := ctx.Req.Method == http.MethodDelete - - m := repoPattern.FindStringSubmatch(path) - if len(m) == 2 && isGetHead { - ctx.SetPathParam("group", strings.Trim(m[1], "/")) - rpm.GetRepositoryConfig(ctx) - return - } - m = repoFilePattern.FindStringSubmatch(path) - if len(m) == 3 && isGetHead { - ctx.SetPathParam("group", strings.Trim(m[1], "/")) - ctx.SetPathParam("filename", m[2]) - if isHead { - rpm.CheckRepositoryFileExistence(ctx) - } else { - rpm.GetRepositoryFile(ctx) - } - return - } - - m = uploadPattern.FindStringSubmatch(path) - if len(m) == 2 && isPut { - reqPackageAccess(perm.AccessModeWrite)(ctx) - if ctx.Written() { - return - } - ctx.SetPathParam("group", strings.Trim(m[1], "/")) - rpm.UploadPackageFile(ctx) - return - } - - m = filePattern.FindStringSubmatch(path) - if len(m) == 6 && (isGetHead || isDelete) { - ctx.SetPathParam("group", strings.Trim(m[1], "/")) - ctx.SetPathParam("name", m[2]) - ctx.SetPathParam("version", m[3]) - ctx.SetPathParam("architecture", m[4]) - if isGetHead { - rpm.DownloadPackageFile(ctx) - } else { - reqPackageAccess(perm.AccessModeWrite)(ctx) - if ctx.Written() { - return - } - rpm.DeletePackageFile(ctx) - } - return - } - - ctx.Status(http.StatusNotFound) - }) + r.Methods("HEAD,GET", "/rpm.repo", reqPackageAccess(perm.AccessModeRead), rpm.GetRepositoryConfig) + r.PathGroup("/rpm/*", func(g *web.RouterPathGroup) { + g.MatchPath("HEAD,GET", "/repository.key", rpm.GetRepositoryKey) + g.MatchPath("HEAD,GET", "/<group:*>.repo", rpm.GetRepositoryConfig) + g.MatchPath("HEAD", "/<group:*>/repodata/<filename>", rpm.CheckRepositoryFileExistence) + g.MatchPath("GET", "/<group:*>/repodata/<filename>", rpm.GetRepositoryFile) + g.MatchPath("PUT", "/<group:*>/upload", reqPackageAccess(perm.AccessModeWrite), rpm.UploadPackageFile) + // this URL pattern is only used internally in the RPM index, it is generated by us, the filename part is not really used (can be anything) + g.MatchPath("HEAD,GET", "/<group:*>/package/<name>/<version>/<architecture>", rpm.DownloadPackageFile) + g.MatchPath("HEAD,GET", "/<group:*>/package/<name>/<version>/<architecture>/<filename>", rpm.DownloadPackageFile) + g.MatchPath("DELETE", "/<group:*>/package/<name>/<version>/<architecture>", reqPackageAccess(perm.AccessModeWrite), rpm.DeletePackageFile) }, reqPackageAccess(perm.AccessModeRead)) + r.Group("/rubygems", func() { r.Get("/specs.4.8.gz", rubygems.EnumeratePackages) r.Get("/latest_specs.4.8.gz", rubygems.EnumeratePackagesLatest) @@ -621,6 +486,7 @@ func CommonRoutes() *web.Router { r.Delete("/yank", rubygems.DeletePackage) }, reqPackageAccess(perm.AccessModeWrite)) }, reqPackageAccess(perm.AccessModeRead)) + r.Group("/swift", func() { r.Group("", func() { // Needs to be unauthenticated. r.Post("", swift.CheckAuthenticate) @@ -632,31 +498,12 @@ func CommonRoutes() *web.Router { r.Get("", swift.EnumeratePackageVersions) r.Get(".json", swift.EnumeratePackageVersions) }, swift.CheckAcceptMediaType(swift.AcceptJSON)) - r.Group("/{version}", func() { - r.Get("/Package.swift", swift.CheckAcceptMediaType(swift.AcceptSwift), swift.DownloadManifest) - r.Put("", reqPackageAccess(perm.AccessModeWrite), swift.CheckAcceptMediaType(swift.AcceptJSON), swift.UploadPackageFile) - r.Get("", func(ctx *context.Context) { - // Can't use normal routes here: https://github.com/go-chi/chi/issues/781 - - version := ctx.PathParam("version") - if strings.HasSuffix(version, ".zip") { - swift.CheckAcceptMediaType(swift.AcceptZip)(ctx) - if ctx.Written() { - return - } - ctx.SetPathParam("version", version[:len(version)-4]) - swift.DownloadPackageFile(ctx) - } else { - swift.CheckAcceptMediaType(swift.AcceptJSON)(ctx) - if ctx.Written() { - return - } - if strings.HasSuffix(version, ".json") { - ctx.SetPathParam("version", version[:len(version)-5]) - } - swift.PackageVersionMetadata(ctx) - } - }) + r.PathGroup("/*", func(g *web.RouterPathGroup) { + g.MatchPath("GET", "/<version>.json", swift.CheckAcceptMediaType(swift.AcceptJSON), swift.PackageVersionMetadata) + g.MatchPath("GET", "/<version>.zip", swift.CheckAcceptMediaType(swift.AcceptZip), swift.DownloadPackageFile) + g.MatchPath("GET", "/<version>/Package.swift", swift.CheckAcceptMediaType(swift.AcceptSwift), swift.DownloadManifest) + g.MatchPath("GET", "/<version>", swift.CheckAcceptMediaType(swift.AcceptJSON), swift.PackageVersionMetadata) + g.MatchPath("PUT", "/<version>", reqPackageAccess(perm.AccessModeWrite), swift.CheckAcceptMediaType(swift.AcceptJSON), swift.UploadPackageFile) }) }) r.Get("/identifiers", swift.CheckAcceptMediaType(swift.AcceptJSON), swift.LookupPackageIdentifiers) @@ -693,6 +540,8 @@ func ContainerRoutes() *web.Router { &container.Auth{}, }) + // TODO: Content Discovery / References (not implemented yet) + r.Get("", container.ReqContainerAccess, container.DetermineSupport) r.Group("/token", func() { r.Get("", container.Authenticate) @@ -701,27 +550,22 @@ func ContainerRoutes() *web.Router { r.Get("/_catalog", container.ReqContainerAccess, container.GetRepositoryList) r.Group("/{username}", func() { r.PathGroup("/*", func(g *web.RouterPathGroup) { - g.MatchPath("POST", "/<image:*>/blobs/uploads", reqPackageAccess(perm.AccessModeWrite), container.VerifyImageName, container.InitiateUploadBlob) - g.MatchPath("GET", "/<image:*>/tags/list", container.VerifyImageName, container.GetTagList) - g.MatchPath("GET,PATCH,PUT,DELETE", `/<image:*>/blobs/uploads/<uuid:[-.=\w]+>`, reqPackageAccess(perm.AccessModeWrite), container.VerifyImageName, func(ctx *context.Context) { - switch ctx.Req.Method { - case http.MethodGet: - container.GetUploadBlob(ctx) - case http.MethodPatch: - container.UploadBlob(ctx) - case http.MethodPut: - container.EndUploadBlob(ctx) - default: /* DELETE */ - container.CancelUploadBlob(ctx) - } - }) + g.MatchPath("POST", "/<image:*>/blobs/uploads", reqPackageAccess(perm.AccessModeWrite), container.VerifyImageName, container.PostBlobsUploads) + g.MatchPath("GET", "/<image:*>/tags/list", container.VerifyImageName, container.GetTagsList) + + patternBlobsUploadsUUID := g.PatternRegexp(`/<image:*>/blobs/uploads/<uuid:[-.=\w]+>`, reqPackageAccess(perm.AccessModeWrite), container.VerifyImageName) + g.MatchPattern("GET", patternBlobsUploadsUUID, container.GetBlobsUpload) + g.MatchPattern("PATCH", patternBlobsUploadsUUID, container.PatchBlobsUpload) + g.MatchPattern("PUT", patternBlobsUploadsUUID, container.PutBlobsUpload) + g.MatchPattern("DELETE", patternBlobsUploadsUUID, container.DeleteBlobsUpload) + g.MatchPath("HEAD", `/<image:*>/blobs/<digest>`, container.VerifyImageName, container.HeadBlob) g.MatchPath("GET", `/<image:*>/blobs/<digest>`, container.VerifyImageName, container.GetBlob) g.MatchPath("DELETE", `/<image:*>/blobs/<digest>`, container.VerifyImageName, reqPackageAccess(perm.AccessModeWrite), container.DeleteBlob) g.MatchPath("HEAD", `/<image:*>/manifests/<reference>`, container.VerifyImageName, container.HeadManifest) g.MatchPath("GET", `/<image:*>/manifests/<reference>`, container.VerifyImageName, container.GetManifest) - g.MatchPath("PUT", `/<image:*>/manifests/<reference>`, container.VerifyImageName, reqPackageAccess(perm.AccessModeWrite), container.UploadManifest) + g.MatchPath("PUT", `/<image:*>/manifests/<reference>`, container.VerifyImageName, reqPackageAccess(perm.AccessModeWrite), container.PutManifest) g.MatchPath("DELETE", `/<image:*>/manifests/<reference>`, container.VerifyImageName, reqPackageAccess(perm.AccessModeWrite), container.DeleteManifest) }) }, container.ReqContainerAccess, context.UserAssignmentWeb(), context.PackageAssignment(), reqPackageAccess(perm.AccessModeRead)) diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index f5dc6c1d01..bf9cc3f1b8 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -239,7 +239,7 @@ func GetPackageOrRepositoryFile(ctx *context.Context) { return } - s, u, pf, err := packages_service.GetPackageFileStream(ctx, pfs[0]) + s, u, pf, err := packages_service.OpenFileForDownload(ctx, pfs[0]) if err != nil { if errors.Is(err, util.ErrNotExist) { apiError(ctx, http.StatusNotFound, err) diff --git a/routers/api/packages/cargo/cargo.go b/routers/api/packages/cargo/cargo.go index 710c614c6e..cfcf79244f 100644 --- a/routers/api/packages/cargo/cargo.go +++ b/routers/api/packages/cargo/cargo.go @@ -95,10 +95,7 @@ type SearchResultMeta struct { // https://doc.rust-lang.org/cargo/reference/registries.html#search func SearchPackages(ctx *context.Context) { - page := ctx.FormInt("page") - if page < 1 { - page = 1 - } + page := max(ctx.FormInt("page"), 1) perPage := ctx.FormInt("per_page") paginator := db.ListOptions{ Page: page, @@ -168,7 +165,7 @@ func ListOwners(ctx *context.Context) { // DownloadPackageFile serves the content of a package func DownloadPackageFile(ctx *context.Context) { - s, u, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + s, u, pf, err := packages_service.OpenFileForDownloadByPackageNameAndVersion( ctx, &packages_service.PackageInfo{ Owner: ctx.Package.Owner, diff --git a/routers/api/packages/chef/chef.go b/routers/api/packages/chef/chef.go index a0c8c5696c..1f11afe548 100644 --- a/routers/api/packages/chef/chef.go +++ b/routers/api/packages/chef/chef.go @@ -343,7 +343,7 @@ func DownloadPackage(ctx *context.Context) { pf := pd.Files[0].File - s, u, _, err := packages_service.GetPackageFileStream(ctx, pf) + s, u, _, err := packages_service.OpenFileForDownload(ctx, pf) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/composer/composer.go b/routers/api/packages/composer/composer.go index c6c14e5cf4..9daf0ffeff 100644 --- a/routers/api/packages/composer/composer.go +++ b/routers/api/packages/composer/composer.go @@ -53,10 +53,7 @@ func ServiceIndex(ctx *context.Context) { // SearchPackages searches packages, only "q" is supported // https://packagist.org/apidoc#search-packages func SearchPackages(ctx *context.Context) { - page := ctx.FormInt("page") - if page < 1 { - page = 1 - } + page := max(ctx.FormInt("page"), 1) perPage := ctx.FormInt("per_page") paginator := db.ListOptions{ Page: page, @@ -163,7 +160,7 @@ func PackageMetadata(ctx *context.Context) { // DownloadPackageFile serves the content of a package func DownloadPackageFile(ctx *context.Context) { - s, u, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + s, u, pf, err := packages_service.OpenFileForDownloadByPackageNameAndVersion( ctx, &packages_service.PackageInfo{ Owner: ctx.Package.Owner, diff --git a/routers/api/packages/conan/conan.go b/routers/api/packages/conan/conan.go index 8019eee9f7..fe70e02cd6 100644 --- a/routers/api/packages/conan/conan.go +++ b/routers/api/packages/conan/conan.go @@ -480,7 +480,7 @@ func downloadFile(ctx *context.Context, fileFilter container.Set[string], fileKe return } - s, u, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + s, u, pf, err := packages_service.OpenFileForDownloadByPackageNameAndVersion( ctx, &packages_service.PackageInfo{ Owner: ctx.Package.Owner, diff --git a/routers/api/packages/conda/conda.go b/routers/api/packages/conda/conda.go index 7a46681235..cfe069d6db 100644 --- a/routers/api/packages/conda/conda.go +++ b/routers/api/packages/conda/conda.go @@ -36,6 +36,24 @@ func apiError(ctx *context.Context, status int, obj any) { }) } +func isCondaPackageFileName(filename string) bool { + return strings.HasSuffix(filename, ".tar.bz2") || strings.HasSuffix(filename, ".conda") +} + +func ListOrGetPackages(ctx *context.Context) { + filename := ctx.PathParam("filename") + switch filename { + case "repodata.json", "repodata.json.bz2", "current_repodata.json", "current_repodata.json.bz2": + EnumeratePackages(ctx) + return + } + if isCondaPackageFileName(filename) { + DownloadPackageFile(ctx) + return + } + ctx.NotFound(nil) +} + func EnumeratePackages(ctx *context.Context) { type Info struct { Subdir string `json:"subdir"` @@ -174,6 +192,12 @@ func EnumeratePackages(ctx *context.Context) { } func UploadPackageFile(ctx *context.Context) { + filename := ctx.PathParam("filename") + if !isCondaPackageFileName(filename) { + apiError(ctx, http.StatusBadRequest, nil) + return + } + upload, needToClose, err := ctx.UploadStream() if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -191,7 +215,7 @@ func UploadPackageFile(ctx *context.Context) { defer buf.Close() var pck *conda_module.Package - if strings.HasSuffix(strings.ToLower(ctx.PathParam("filename")), ".tar.bz2") { + if strings.HasSuffix(filename, ".tar.bz2") { pck, err = conda_module.ParsePackageBZ2(buf) } else { pck, err = conda_module.ParsePackageConda(buf, buf.Size()) @@ -293,7 +317,7 @@ func DownloadPackageFile(ctx *context.Context) { pf := pfs[0] - s, u, _, err := packages_service.GetPackageFileStream(ctx, pf) + s, u, _, err := packages_service.OpenFileForDownload(ctx, pf) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/container/auth.go b/routers/api/packages/container/auth.go index 1d8ae6af7d..1e1b87eb79 100644 --- a/routers/api/packages/container/auth.go +++ b/routers/api/packages/container/auth.go @@ -21,7 +21,7 @@ func (a *Auth) Name() string { } // Verify extracts the user from the Bearer token -// If it's an anonymous session a ghost user is returned +// If it's an anonymous session, a ghost user is returned func (a *Auth) Verify(req *http.Request, w http.ResponseWriter, store auth.DataStore, sess auth.SessionStore) (*user_model.User, error) { packageMeta, err := packages.ParseAuthorizationRequest(req) if err != nil { diff --git a/routers/api/packages/container/blob.go b/routers/api/packages/container/blob.go index 4a2320ab76..4b7bcee9d0 100644 --- a/routers/api/packages/container/blob.go +++ b/routers/api/packages/container/blob.go @@ -20,11 +20,13 @@ import ( container_module "code.gitea.io/gitea/modules/packages/container" "code.gitea.io/gitea/modules/util" packages_service "code.gitea.io/gitea/services/packages" + + "github.com/opencontainers/go-digest" ) // saveAsPackageBlob creates a package blob from an upload // The uploaded blob gets stored in a special upload version to link them to the package/image -func saveAsPackageBlob(ctx context.Context, hsr packages_module.HashedSizeReader, pci *packages_service.PackageCreationInfo) (*packages_model.PackageBlob, error) { //nolint:unparam +func saveAsPackageBlob(ctx context.Context, hsr packages_module.HashedSizeReader, pci *packages_service.PackageCreationInfo) (*packages_model.PackageBlob, error) { //nolint:unparam // PackageBlob is never used pb := packages_service.NewPackageBlob(hsr) exists := false @@ -88,20 +90,18 @@ func mountBlob(ctx context.Context, pi *packages_service.PackageInfo, pb *packag }) } -func containerPkgName(piOwnerID int64, piName string) string { - return fmt.Sprintf("pkg_%d_container_%s", piOwnerID, strings.ToLower(piName)) +func containerGlobalLockKey(piOwnerID int64, piName, usage string) string { + return fmt.Sprintf("pkg_%d_container_%s_%s", piOwnerID, strings.ToLower(piName), usage) } func getOrCreateUploadVersion(ctx context.Context, pi *packages_service.PackageInfo) (*packages_model.PackageVersion, error) { - var uploadVersion *packages_model.PackageVersion - - releaser, err := globallock.Lock(ctx, containerPkgName(pi.Owner.ID, pi.Name)) + releaser, err := globallock.Lock(ctx, containerGlobalLockKey(pi.Owner.ID, pi.Name, "package")) if err != nil { return nil, err } defer releaser() - err = db.WithTx(ctx, func(ctx context.Context) error { + return db.WithTx2(ctx, func(ctx context.Context) (*packages_model.PackageVersion, error) { created := true p := &packages_model.Package{ OwnerID: pi.Owner.ID, @@ -113,7 +113,7 @@ func getOrCreateUploadVersion(ctx context.Context, pi *packages_service.PackageI if p, err = packages_model.TryInsertPackage(ctx, p); err != nil { if !errors.Is(err, packages_model.ErrDuplicatePackage) { log.Error("Error inserting package: %v", err) - return err + return nil, err } created = false } @@ -121,31 +121,26 @@ func getOrCreateUploadVersion(ctx context.Context, pi *packages_service.PackageI if created { if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository, strings.ToLower(pi.Owner.LowerName+"/"+pi.Name)); err != nil { log.Error("Error setting package property: %v", err) - return err + return nil, err } } pv := &packages_model.PackageVersion{ PackageID: p.ID, CreatorID: pi.Owner.ID, - Version: container_model.UploadVersion, - LowerVersion: container_model.UploadVersion, + Version: container_module.UploadVersion, + LowerVersion: container_module.UploadVersion, IsInternal: true, MetadataJSON: "null", } if pv, err = packages_model.GetOrInsertVersion(ctx, pv); err != nil { if !errors.Is(err, packages_model.ErrDuplicatePackageVersion) { log.Error("Error inserting package: %v", err) - return err + return nil, err } } - - uploadVersion = pv - - return nil + return pv, nil }) - - return uploadVersion, err } func createFileForBlob(ctx context.Context, pv *packages_model.PackageVersion, pb *packages_model.PackageBlob) error { @@ -175,8 +170,8 @@ func createFileForBlob(ctx context.Context, pv *packages_model.PackageVersion, p return nil } -func deleteBlob(ctx context.Context, ownerID int64, image, digest string) error { - releaser, err := globallock.Lock(ctx, containerPkgName(ownerID, image)) +func deleteBlob(ctx context.Context, ownerID int64, image string, digest digest.Digest) error { + releaser, err := globallock.Lock(ctx, containerGlobalLockKey(ownerID, image, "blob")) if err != nil { return err } @@ -186,7 +181,7 @@ func deleteBlob(ctx context.Context, ownerID int64, image, digest string) error pfds, err := container_model.GetContainerBlobs(ctx, &container_model.BlobSearchOptions{ OwnerID: ownerID, Image: image, - Digest: digest, + Digest: string(digest), }) if err != nil { return err diff --git a/routers/api/packages/container/container.go b/routers/api/packages/container/container.go index 6ef1655235..d532f698ad 100644 --- a/routers/api/packages/container/container.go +++ b/routers/api/packages/container/container.go @@ -13,6 +13,7 @@ import ( "regexp" "strconv" "strings" + "sync" auth_model "code.gitea.io/gitea/models/auth" packages_model "code.gitea.io/gitea/models/packages" @@ -21,6 +22,7 @@ import ( "code.gitea.io/gitea/modules/httplib" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/optional" packages_module "code.gitea.io/gitea/modules/packages" container_module "code.gitea.io/gitea/modules/packages/container" "code.gitea.io/gitea/modules/setting" @@ -31,17 +33,21 @@ import ( packages_service "code.gitea.io/gitea/services/packages" container_service "code.gitea.io/gitea/services/packages/container" - digest "github.com/opencontainers/go-digest" + "github.com/opencontainers/go-digest" ) // maximum size of a container manifest // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-manifests const maxManifestSize = 10 * 1024 * 1024 -var ( - imageNamePattern = regexp.MustCompile(`\A[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*\z`) - referencePattern = regexp.MustCompile(`\A[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}\z`) -) +var globalVars = sync.OnceValue(func() (ret struct { + imageNamePattern, referencePattern *regexp.Regexp +}, +) { + ret.imageNamePattern = regexp.MustCompile(`\A[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*\z`) + ret.referencePattern = regexp.MustCompile(`\A[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}\z`) + return ret +}) type containerHeaders struct { Status int @@ -50,7 +56,7 @@ type containerHeaders struct { Range string Location string ContentType string - ContentLength int64 + ContentLength optional.Option[int64] } // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#legacy-docker-support-http-headers @@ -64,8 +70,8 @@ func setResponseHeaders(resp http.ResponseWriter, h *containerHeaders) { if h.ContentType != "" { resp.Header().Set("Content-Type", h.ContentType) } - if h.ContentLength != 0 { - resp.Header().Set("Content-Length", strconv.FormatInt(h.ContentLength, 10)) + if h.ContentLength.Has() { + resp.Header().Set("Content-Length", strconv.FormatInt(h.ContentLength.Value(), 10)) } if h.UploadUUID != "" { resp.Header().Set("Docker-Upload-Uuid", h.UploadUUID) @@ -83,9 +89,7 @@ func jsonResponse(ctx *context.Context, status int, obj any) { Status: status, ContentType: "application/json", }) - if err := json.NewEncoder(ctx.Resp).Encode(obj); err != nil { - log.Error("JSON encode: %v", err) - } + _ = json.NewEncoder(ctx.Resp).Encode(obj) // ignore network errors } func apiError(ctx *context.Context, status int, err error) { @@ -133,7 +137,7 @@ func ReqContainerAccess(ctx *context.Context) { // VerifyImageName is a middleware which checks if the image name is allowed func VerifyImageName(ctx *context.Context) { - if !imageNamePattern.MatchString(ctx.PathParam("image")) { + if !globalVars().imageNamePattern.MatchString(ctx.PathParam("image")) { apiErrorDefined(ctx, errNameInvalid) } } @@ -215,7 +219,7 @@ func GetRepositoryList(ctx *context.Context) { if len(repositories) == n { v := url.Values{} if n > 0 { - v.Add("n", strconv.Itoa(n)) + v.Add("n", strconv.Itoa(n)) // FIXME: "n" can't be zero here, the logic is inconsistent with GetTagsList } v.Add("last", repositories[len(repositories)-1]) @@ -230,7 +234,7 @@ func GetRepositoryList(ctx *context.Context) { // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#mounting-a-blob-from-another-repository // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#single-post // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-a-blob-in-chunks -func InitiateUploadBlob(ctx *context.Context) { +func PostBlobsUploads(ctx *context.Context) { image := ctx.PathParam("image") mount := ctx.FormTrim("mount") @@ -312,14 +316,14 @@ func InitiateUploadBlob(ctx *context.Context) { setResponseHeaders(ctx.Resp, &containerHeaders{ Location: fmt.Sprintf("/v2/%s/%s/blobs/uploads/%s", ctx.Package.Owner.LowerName, image, upload.ID), - Range: "0-0", UploadUUID: upload.ID, Status: http.StatusAccepted, }) } -// https://docs.docker.com/registry/spec/api/#get-blob-upload -func GetUploadBlob(ctx *context.Context) { +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-a-blob-in-chunks +func GetBlobsUpload(ctx *context.Context) { + image := ctx.PathParam("image") uuid := ctx.PathParam("uuid") upload, err := packages_model.GetBlobUploadByID(ctx, uuid) @@ -332,15 +336,21 @@ func GetUploadBlob(ctx *context.Context) { return } - setResponseHeaders(ctx.Resp, &containerHeaders{ - Range: fmt.Sprintf("0-%d", upload.BytesReceived), + // FIXME: undefined behavior when the uploaded content is empty: https://github.com/opencontainers/distribution-spec/issues/578 + respHeaders := &containerHeaders{ + Location: fmt.Sprintf("/v2/%s/%s/blobs/uploads/%s", ctx.Package.Owner.LowerName, image, upload.ID), UploadUUID: upload.ID, Status: http.StatusNoContent, - }) + } + if upload.BytesReceived > 0 { + respHeaders.Range = fmt.Sprintf("0-%d", upload.BytesReceived-1) + } + setResponseHeaders(ctx.Resp, respHeaders) } +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#single-post // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-a-blob-in-chunks -func UploadBlob(ctx *context.Context) { +func PatchBlobsUpload(ctx *context.Context) { image := ctx.PathParam("image") uploader, err := container_service.NewBlobUploader(ctx, ctx.PathParam("uuid")) @@ -376,16 +386,19 @@ func UploadBlob(ctx *context.Context) { return } - setResponseHeaders(ctx.Resp, &containerHeaders{ + respHeaders := &containerHeaders{ Location: fmt.Sprintf("/v2/%s/%s/blobs/uploads/%s", ctx.Package.Owner.LowerName, image, uploader.ID), - Range: fmt.Sprintf("0-%d", uploader.Size()-1), UploadUUID: uploader.ID, Status: http.StatusAccepted, - }) + } + if uploader.Size() > 0 { + respHeaders.Range = fmt.Sprintf("0-%d", uploader.Size()-1) + } + setResponseHeaders(ctx.Resp, respHeaders) } // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-a-blob-in-chunks -func EndUploadBlob(ctx *context.Context) { +func PutBlobsUpload(ctx *context.Context) { image := ctx.PathParam("image") digest := ctx.FormTrim("digest") @@ -403,12 +416,7 @@ func EndUploadBlob(ctx *context.Context) { } return } - doClose := true - defer func() { - if doClose { - uploader.Close() - } - }() + defer uploader.Close() if ctx.Req.Body != nil { if err := uploader.Append(ctx, ctx.Req.Body); err != nil { @@ -441,11 +449,10 @@ func EndUploadBlob(ctx *context.Context) { return } - if err := uploader.Close(); err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - doClose = false + // There was a strange bug: the "Close" fails with error "close .../tmp/package-upload/....: file already closed" + // AFAIK there should be no other "Close" call to the uploader between NewBlobUploader and this line. + // At least it's safe to call Close twice, so ignore the error. + _ = uploader.Close() if err := container_service.RemoveBlobUploadByID(ctx, uploader.ID); err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -460,7 +467,7 @@ func EndUploadBlob(ctx *context.Context) { } // https://docs.docker.com/registry/spec/api/#delete-blob-upload -func CancelUploadBlob(ctx *context.Context) { +func DeleteBlobsUpload(ctx *context.Context) { uuid := ctx.PathParam("uuid") _, err := packages_model.GetBlobUploadByID(ctx, uuid) @@ -484,16 +491,15 @@ func CancelUploadBlob(ctx *context.Context) { } func getBlobFromContext(ctx *context.Context) (*packages_model.PackageFileDescriptor, error) { - d := ctx.PathParam("digest") - - if digest.Digest(d).Validate() != nil { + d := digest.Digest(ctx.PathParam("digest")) + if d.Validate() != nil { return nil, container_model.ErrContainerBlobNotExist } return workaroundGetContainerBlob(ctx, &container_model.BlobSearchOptions{ OwnerID: ctx.Package.Owner.ID, Image: ctx.PathParam("image"), - Digest: d, + Digest: string(d), }) } @@ -511,7 +517,7 @@ func HeadBlob(ctx *context.Context) { setResponseHeaders(ctx.Resp, &containerHeaders{ ContentDigest: blob.Properties.GetByName(container_module.PropertyDigest), - ContentLength: blob.Blob.Size, + ContentLength: optional.Some(blob.Blob.Size), Status: http.StatusOK, }) } @@ -533,9 +539,8 @@ func GetBlob(ctx *context.Context) { // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#deleting-blobs func DeleteBlob(ctx *context.Context) { - d := ctx.PathParam("digest") - - if digest.Digest(d).Validate() != nil { + d := digest.Digest(ctx.PathParam("digest")) + if d.Validate() != nil { apiErrorDefined(ctx, errBlobUnknown) return } @@ -551,7 +556,7 @@ func DeleteBlob(ctx *context.Context) { } // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-manifests -func UploadManifest(ctx *context.Context) { +func PutManifest(ctx *context.Context) { reference := ctx.PathParam("reference") mci := &manifestCreationInfo{ @@ -563,7 +568,7 @@ func UploadManifest(ctx *context.Context) { IsTagged: digest.Digest(reference).Validate() != nil, } - if mci.IsTagged && !referencePattern.MatchString(reference) { + if mci.IsTagged && !globalVars().referencePattern.MatchString(reference) { apiErrorDefined(ctx, errManifestInvalid.WithMessage("Tag is invalid")) return } @@ -607,18 +612,18 @@ func UploadManifest(ctx *context.Context) { } func getBlobSearchOptionsFromContext(ctx *context.Context) (*container_model.BlobSearchOptions, error) { - reference := ctx.PathParam("reference") - opts := &container_model.BlobSearchOptions{ OwnerID: ctx.Package.Owner.ID, Image: ctx.PathParam("image"), IsManifest: true, } - if digest.Digest(reference).Validate() == nil { - opts.Digest = reference - } else if referencePattern.MatchString(reference) { + reference := ctx.PathParam("reference") + if d := digest.Digest(reference); d.Validate() == nil { + opts.Digest = string(d) + } else if globalVars().referencePattern.MatchString(reference) { opts.Tag = reference + opts.OnlyLead = true } else { return nil, container_model.ErrContainerBlobNotExist } @@ -650,7 +655,7 @@ func HeadManifest(ctx *context.Context) { setResponseHeaders(ctx.Resp, &containerHeaders{ ContentDigest: manifest.Properties.GetByName(container_module.PropertyDigest), ContentType: manifest.Properties.GetByName(container_module.PropertyMediaType), - ContentLength: manifest.Blob.Size, + ContentLength: optional.Some(manifest.Blob.Size), Status: http.StatusOK, }) } @@ -705,7 +710,7 @@ func DeleteManifest(ctx *context.Context) { func serveBlob(ctx *context.Context, pfd *packages_model.PackageFileDescriptor) { serveDirectReqParams := make(url.Values) serveDirectReqParams.Set("response-content-type", pfd.Properties.GetByName(container_module.PropertyMediaType)) - s, u, _, err := packages_service.GetPackageBlobStream(ctx, pfd.File, pfd.Blob, serveDirectReqParams) + s, u, _, err := packages_service.OpenBlobForDownload(ctx, pfd.File, pfd.Blob, serveDirectReqParams) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -714,14 +719,14 @@ func serveBlob(ctx *context.Context, pfd *packages_model.PackageFileDescriptor) headers := &containerHeaders{ ContentDigest: pfd.Properties.GetByName(container_module.PropertyDigest), ContentType: pfd.Properties.GetByName(container_module.PropertyMediaType), - ContentLength: pfd.Blob.Size, + ContentLength: optional.Some(pfd.Blob.Size), Status: http.StatusOK, } if u != nil { headers.Status = http.StatusTemporaryRedirect headers.Location = u.String() - + headers.ContentLength = optional.None[int64]() // do not set Content-Length for redirect responses setResponseHeaders(ctx.Resp, headers) return } @@ -735,7 +740,7 @@ func serveBlob(ctx *context.Context, pfd *packages_model.PackageFileDescriptor) } // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#content-discovery -func GetTagList(ctx *context.Context) { +func GetTagsList(ctx *context.Context) { image := ctx.PathParam("image") if _, err := packages_model.GetPackageByName(ctx, ctx.Package.Owner.ID, packages_model.TypeContainer, image); err != nil { @@ -780,7 +785,8 @@ func GetTagList(ctx *context.Context) { }) } -// FIXME: Workaround to be removed in v1.20 +// FIXME: Workaround to be removed in v1.20. +// Update maybe we should never really remote it, as long as there is legacy data? // https://github.com/go-gitea/gitea/issues/19586 func workaroundGetContainerBlob(ctx *context.Context, opts *container_model.BlobSearchOptions) (*packages_model.PackageFileDescriptor, error) { blob, err := container_model.GetContainerBlob(ctx, opts) diff --git a/routers/api/packages/container/manifest.go b/routers/api/packages/container/manifest.go index 26faa7b024..de40215aa7 100644 --- a/routers/api/packages/container/manifest.go +++ b/routers/api/packages/container/manifest.go @@ -10,11 +10,13 @@ import ( "io" "os" "strings" + "time" "code.gitea.io/gitea/models/db" packages_model "code.gitea.io/gitea/models/packages" container_model "code.gitea.io/gitea/models/packages/container" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/globallock" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" packages_module "code.gitea.io/gitea/modules/packages" @@ -22,23 +24,12 @@ import ( "code.gitea.io/gitea/modules/util" notify_service "code.gitea.io/gitea/services/notify" packages_service "code.gitea.io/gitea/services/packages" + container_service "code.gitea.io/gitea/services/packages/container" - digest "github.com/opencontainers/go-digest" + "github.com/opencontainers/go-digest" oci "github.com/opencontainers/image-spec/specs-go/v1" ) -func isValidMediaType(mt string) bool { - return strings.HasPrefix(mt, "application/vnd.docker.") || strings.HasPrefix(mt, "application/vnd.oci.") -} - -func isImageManifestMediaType(mt string) bool { - return strings.EqualFold(mt, oci.MediaTypeImageManifest) || strings.EqualFold(mt, "application/vnd.docker.distribution.manifest.v2+json") -} - -func isImageIndexMediaType(mt string) bool { - return strings.EqualFold(mt, oci.MediaTypeImageIndex) || strings.EqualFold(mt, "application/vnd.docker.distribution.manifest.list.v2+json") -} - // manifestCreationInfo describes a manifest to create type manifestCreationInfo struct { MediaType string @@ -55,71 +46,71 @@ func processManifest(ctx context.Context, mci *manifestCreationInfo, buf *packag if err := json.NewDecoder(buf).Decode(&index); err != nil { return "", err } - if index.SchemaVersion != 2 { return "", errUnsupported.WithMessage("Schema version is not supported") } - if _, err := buf.Seek(0, io.SeekStart); err != nil { return "", err } - if !isValidMediaType(mci.MediaType) { + if !container_module.IsMediaTypeValid(mci.MediaType) { mci.MediaType = index.MediaType - if !isValidMediaType(mci.MediaType) { + if !container_module.IsMediaTypeValid(mci.MediaType) { return "", errManifestInvalid.WithMessage("MediaType not recognized") } } - if isImageManifestMediaType(mci.MediaType) { - return processImageManifest(ctx, mci, buf) - } else if isImageIndexMediaType(mci.MediaType) { - return processImageManifestIndex(ctx, mci, buf) + // .../container/manifest.go:453:createManifestBlob() [E] Error inserting package blob: Error 1062 (23000): Duplicate entry '..........' for key 'package_blob.UQE_package_blob_md5' + releaser, err := globallock.Lock(ctx, containerGlobalLockKey(mci.Owner.ID, mci.Image, "manifest")) + if err != nil { + return "", err + } + defer releaser() + + if container_module.IsMediaTypeImageManifest(mci.MediaType) { + return processOciImageManifest(ctx, mci, buf) + } else if container_module.IsMediaTypeImageIndex(mci.MediaType) { + return processOciImageIndex(ctx, mci, buf) } return "", errManifestInvalid } -func processImageManifest(ctx context.Context, mci *manifestCreationInfo, buf *packages_module.HashedBuffer) (string, error) { - manifestDigest := "" - - err := func() error { - var manifest oci.Manifest - if err := json.NewDecoder(buf).Decode(&manifest); err != nil { - return err - } - - if _, err := buf.Seek(0, io.SeekStart); err != nil { - return err - } - - ctx, committer, err := db.TxContext(ctx) - if err != nil { - return err - } - defer committer.Close() - - configDescriptor, err := container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{ - OwnerID: mci.Owner.ID, - Image: mci.Image, - Digest: string(manifest.Config.Digest), - }) - if err != nil { - return err - } +type processManifestTxRet struct { + pv *packages_model.PackageVersion + pb *packages_model.PackageBlob + created bool + digest string +} - configReader, err := packages_module.NewContentStore().Get(packages_module.BlobHash256Key(configDescriptor.Blob.HashSHA256)) - if err != nil { - return err +func handleCreateManifestResult(ctx context.Context, err error, mci *manifestCreationInfo, contentStore *packages_module.ContentStore, txRet *processManifestTxRet) (string, error) { + if err != nil && txRet.created && txRet.pb != nil { + if err := contentStore.Delete(packages_module.BlobHash256Key(txRet.pb.HashSHA256)); err != nil { + log.Error("Error deleting package blob from content store: %v", err) } - defer configReader.Close() + return "", err + } + pd, err := packages_model.GetPackageDescriptor(ctx, txRet.pv) + if err != nil { + log.Error("Error getting package descriptor: %v", err) // ignore this error + } else { + notify_service.PackageCreate(ctx, mci.Creator, pd) + } + return txRet.digest, nil +} - metadata, err := container_module.ParseImageConfig(manifest.Config.MediaType, configReader) - if err != nil { - return err - } +func processOciImageManifest(ctx context.Context, mci *manifestCreationInfo, buf *packages_module.HashedBuffer) (manifestDigest string, errRet error) { + manifest, configDescriptor, metadata, err := container_service.ParseManifestMetadata(ctx, buf, mci.Owner.ID, mci.Image) + if err != nil { + return "", err + } + if _, err = buf.Seek(0, io.SeekStart); err != nil { + return "", err + } + contentStore := packages_module.NewContentStore() + var txRet processManifestTxRet + err = db.WithTx(ctx, func(ctx context.Context) (err error) { blobReferences := make([]*blobReference, 0, 1+len(manifest.Layers)) - blobReferences = append(blobReferences, &blobReference{ Digest: manifest.Config.Digest, MediaType: manifest.Config.MediaType, @@ -150,78 +141,43 @@ func processImageManifest(ctx context.Context, mci *manifestCreationInfo, buf *p return err } - uploadVersion, err := packages_model.GetInternalVersionByNameAndVersion(ctx, mci.Owner.ID, packages_model.TypeContainer, mci.Image, container_model.UploadVersion) - if err != nil && err != packages_model.ErrPackageNotExist { + uploadVersion, err := packages_model.GetInternalVersionByNameAndVersion(ctx, mci.Owner.ID, packages_model.TypeContainer, mci.Image, container_module.UploadVersion) + if err != nil && !errors.Is(err, packages_model.ErrPackageNotExist) { return err } for _, ref := range blobReferences { - if err := createFileFromBlobReference(ctx, pv, uploadVersion, ref); err != nil { + if _, err = createFileFromBlobReference(ctx, pv, uploadVersion, ref); err != nil { return err } } + txRet.pv = pv + txRet.pb, txRet.created, txRet.digest, err = createManifestBlob(ctx, contentStore, mci, pv, buf) + return err + }) - pb, created, digest, err := createManifestBlob(ctx, mci, pv, buf) - removeBlob := false - defer func() { - if removeBlob { - contentStore := packages_module.NewContentStore() - if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil { - log.Error("Error deleting package blob from content store: %v", err) - } - } - }() - if err != nil { - removeBlob = created - return err - } - - if err := committer.Commit(); err != nil { - removeBlob = created - return err - } - - if err := notifyPackageCreate(ctx, mci.Creator, pv); err != nil { - return err - } - - manifestDigest = digest + return handleCreateManifestResult(ctx, err, mci, contentStore, &txRet) +} - return nil - }() - if err != nil { +func processOciImageIndex(ctx context.Context, mci *manifestCreationInfo, buf *packages_module.HashedBuffer) (manifestDigest string, errRet error) { + var index oci.Index + if err := json.NewDecoder(buf).Decode(&index); err != nil { + return "", err + } + if _, err := buf.Seek(0, io.SeekStart); err != nil { return "", err } - return manifestDigest, nil -} - -func processImageManifestIndex(ctx context.Context, mci *manifestCreationInfo, buf *packages_module.HashedBuffer) (string, error) { - manifestDigest := "" - - err := func() error { - var index oci.Index - if err := json.NewDecoder(buf).Decode(&index); err != nil { - return err - } - - if _, err := buf.Seek(0, io.SeekStart); err != nil { - return err - } - - ctx, committer, err := db.TxContext(ctx) - if err != nil { - return err - } - defer committer.Close() - + contentStore := packages_module.NewContentStore() + var txRet processManifestTxRet + err := db.WithTx(ctx, func(ctx context.Context) (err error) { metadata := &container_module.Metadata{ Type: container_module.TypeOCI, Manifests: make([]*container_module.Manifest, 0, len(index.Manifests)), } for _, manifest := range index.Manifests { - if !isImageManifestMediaType(manifest.MediaType) { + if !container_module.IsMediaTypeImageManifest(manifest.MediaType) { return errManifestInvalid } @@ -265,50 +221,12 @@ func processImageManifestIndex(ctx context.Context, mci *manifestCreationInfo, b return err } - pb, created, digest, err := createManifestBlob(ctx, mci, pv, buf) - removeBlob := false - defer func() { - if removeBlob { - contentStore := packages_module.NewContentStore() - if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil { - log.Error("Error deleting package blob from content store: %v", err) - } - } - }() - if err != nil { - removeBlob = created - return err - } - - if err := committer.Commit(); err != nil { - removeBlob = created - return err - } - - if err := notifyPackageCreate(ctx, mci.Creator, pv); err != nil { - return err - } - - manifestDigest = digest - - return nil - }() - if err != nil { - return "", err - } - - return manifestDigest, nil -} - -func notifyPackageCreate(ctx context.Context, doer *user_model.User, pv *packages_model.PackageVersion) error { - pd, err := packages_model.GetPackageDescriptor(ctx, pv) - if err != nil { + txRet.pv = pv + txRet.pb, txRet.created, txRet.digest, err = createManifestBlob(ctx, contentStore, mci, pv, buf) return err - } - - notify_service.PackageCreate(ctx, doer, pd) + }) - return nil + return handleCreateManifestResult(ctx, err, mci, contentStore, &txRet) } func createPackageAndVersion(ctx context.Context, mci *manifestCreationInfo, metadata *container_module.Metadata) (*packages_model.PackageVersion, error) { @@ -349,24 +267,31 @@ func createPackageAndVersion(ctx context.Context, mci *manifestCreationInfo, met LowerVersion: strings.ToLower(mci.Reference), MetadataJSON: string(metadataJSON), } - var pv *packages_model.PackageVersion - if pv, err = packages_model.GetOrInsertVersion(ctx, _pv); err != nil { + pv, err := packages_model.GetOrInsertVersion(ctx, _pv) + if err != nil { if !errors.Is(err, packages_model.ErrDuplicatePackageVersion) { log.Error("Error inserting package: %v", err) return nil, err } - if err = packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil { - return nil, err - } - - // keep download count on overwrite - _pv.DownloadCount = pv.DownloadCount - - if pv, err = packages_model.GetOrInsertVersion(ctx, _pv); err != nil { - if !errors.Is(err, packages_model.ErrDuplicatePackageVersion) { - log.Error("Error inserting package: %v", err) - return nil, err + if container_module.IsMediaTypeImageIndex(mci.MediaType) { + if pv.CreatedUnix.AsTime().Before(time.Now().Add(-24 * time.Hour)) { + if err = packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil { + return nil, err + } + // keep download count on overwriting + _pv.DownloadCount = pv.DownloadCount + if pv, err = packages_model.GetOrInsertVersion(ctx, _pv); err != nil { + if !errors.Is(err, packages_model.ErrDuplicatePackageVersion) { + log.Error("Error inserting package: %v", err) + return nil, err + } + } + } else { + err = packages_model.UpdateVersion(ctx, &packages_model.PackageVersion{ID: pv.ID, MetadataJSON: _pv.MetadataJSON}) + if err != nil { + return nil, err + } } } } @@ -376,14 +301,20 @@ func createPackageAndVersion(ctx context.Context, mci *manifestCreationInfo, met } if mci.IsTagged { - if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestTagged, ""); err != nil { - log.Error("Error setting package version property: %v", err) + if err = packages_model.InsertOrUpdateProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestTagged, ""); err != nil { + return nil, err + } + } else { + if err = packages_model.DeletePropertiesByName(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestTagged); err != nil { return nil, err } } + + if err = packages_model.DeletePropertiesByName(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestReference); err != nil { + return nil, err + } for _, manifest := range metadata.Manifests { - if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestReference, manifest.Digest); err != nil { - log.Error("Error setting package version property: %v", err) + if _, err = packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestReference, manifest.Digest); err != nil { return nil, err } } @@ -400,9 +331,9 @@ type blobReference struct { IsLead bool } -func createFileFromBlobReference(ctx context.Context, pv, uploadVersion *packages_model.PackageVersion, ref *blobReference) error { +func createFileFromBlobReference(ctx context.Context, pv, uploadVersion *packages_model.PackageVersion, ref *blobReference) (*packages_model.PackageFile, error) { if ref.File.Blob.Size != ref.ExpectedSize { - return errSizeInvalid + return nil, errSizeInvalid } if ref.Name == "" { @@ -410,20 +341,21 @@ func createFileFromBlobReference(ctx context.Context, pv, uploadVersion *package } pf := &packages_model.PackageFile{ - VersionID: pv.ID, - BlobID: ref.File.Blob.ID, - Name: ref.Name, - LowerName: ref.Name, - IsLead: ref.IsLead, + VersionID: pv.ID, + BlobID: ref.File.Blob.ID, + Name: ref.Name, + LowerName: ref.Name, + CompositeKey: string(ref.Digest), + IsLead: ref.IsLead, } var err error if pf, err = packages_model.TryInsertFile(ctx, pf); err != nil { if errors.Is(err, packages_model.ErrDuplicatePackageFile) { // Skip this blob because the manifest contains the same filesystem layer multiple times. - return nil + return pf, nil } log.Error("Error inserting package file: %v", err) - return err + return nil, err } props := map[string]string{ @@ -433,21 +365,21 @@ func createFileFromBlobReference(ctx context.Context, pv, uploadVersion *package for name, value := range props { if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeFile, pf.ID, name, value); err != nil { log.Error("Error setting package file property: %v", err) - return err + return nil, err } } - // Remove the file from the blob upload version + // Remove the ref file (old file) from the blob upload version if uploadVersion != nil && ref.File.File != nil && uploadVersion.ID == ref.File.File.VersionID { if err := packages_service.DeletePackageFile(ctx, ref.File.File); err != nil { - return err + return nil, err } } - return nil + return pf, nil } -func createManifestBlob(ctx context.Context, mci *manifestCreationInfo, pv *packages_model.PackageVersion, buf *packages_module.HashedBuffer) (*packages_model.PackageBlob, bool, string, error) { +func createManifestBlob(ctx context.Context, contentStore *packages_module.ContentStore, mci *manifestCreationInfo, pv *packages_model.PackageVersion, buf *packages_module.HashedBuffer) (_ *packages_model.PackageBlob, created bool, manifestDigest string, _ error) { pb, exists, err := packages_model.GetOrInsertBlob(ctx, packages_service.NewPackageBlob(buf)) if err != nil { log.Error("Error inserting package blob: %v", err) @@ -456,29 +388,48 @@ func createManifestBlob(ctx context.Context, mci *manifestCreationInfo, pv *pack // FIXME: Workaround to be removed in v1.20 // https://github.com/go-gitea/gitea/issues/19586 if exists { - err = packages_module.NewContentStore().Has(packages_module.BlobHash256Key(pb.HashSHA256)) + err = contentStore.Has(packages_module.BlobHash256Key(pb.HashSHA256)) if err != nil && (errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist)) { log.Debug("Package registry inconsistent: blob %s does not exist on file system", pb.HashSHA256) exists = false } } if !exists { - contentStore := packages_module.NewContentStore() if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), buf, buf.Size()); err != nil { log.Error("Error saving package blob in content store: %v", err) return nil, false, "", err } } - manifestDigest := digestFromHashSummer(buf) - err = createFileFromBlobReference(ctx, pv, nil, &blobReference{ + manifestDigest = digestFromHashSummer(buf) + pf, err := createFileFromBlobReference(ctx, pv, nil, &blobReference{ Digest: digest.Digest(manifestDigest), MediaType: mci.MediaType, - Name: container_model.ManifestFilename, + Name: container_module.ManifestFilename, File: &packages_model.PackageFileDescriptor{Blob: pb}, ExpectedSize: pb.Size, IsLead: true, }) + if err != nil { + return nil, false, "", err + } + oldManifestFiles, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{ + OwnerID: mci.Owner.ID, + PackageType: packages_model.TypeContainer, + VersionID: pv.ID, + Query: container_module.ManifestFilename, + }) + if err != nil { + return nil, false, "", err + } + for _, oldManifestFile := range oldManifestFiles { + if oldManifestFile.ID != pf.ID && oldManifestFile.IsLead { + err = packages_model.UpdateFile(ctx, &packages_model.PackageFile{ID: oldManifestFile.ID, IsLead: false}, []string{"is_lead"}) + if err != nil { + return nil, false, "", err + } + } + } return pb, !exists, manifestDigest, err } diff --git a/routers/api/packages/cran/cran.go b/routers/api/packages/cran/cran.go index 8a20072cb6..732acd215f 100644 --- a/routers/api/packages/cran/cran.go +++ b/routers/api/packages/cran/cran.go @@ -250,7 +250,7 @@ func downloadPackageFile(ctx *context.Context, opts *cran_model.SearchOptions) { return } - s, u, _, err := packages_service.GetPackageFileStream(ctx, pf) + s, u, _, err := packages_service.OpenFileForDownload(ctx, pf) if err != nil { if errors.Is(err, util.ErrNotExist) { apiError(ctx, http.StatusNotFound, err) diff --git a/routers/api/packages/debian/debian.go b/routers/api/packages/debian/debian.go index fec34c91a6..346f71fa5d 100644 --- a/routers/api/packages/debian/debian.go +++ b/routers/api/packages/debian/debian.go @@ -59,7 +59,7 @@ func GetRepositoryFile(ctx *context.Context) { key += "|" + component + "|" + architecture } - s, u, pf, err := packages_service.GetFileStreamByPackageVersion( + s, u, pf, err := packages_service.OpenFileForDownloadByPackageVersion( ctx, pv, &packages_service.PackageFileInfo{ @@ -106,7 +106,7 @@ func GetRepositoryFileByHash(ctx *context.Context) { return } - s, u, pf, err := packages_service.GetPackageFileStream(ctx, pfs[0]) + s, u, pf, err := packages_service.OpenFileForDownload(ctx, pfs[0]) if err != nil { if errors.Is(err, util.ErrNotExist) { apiError(ctx, http.StatusNotFound, err) @@ -210,7 +210,7 @@ func DownloadPackageFile(ctx *context.Context) { name := ctx.PathParam("name") version := ctx.PathParam("version") - s, u, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + s, u, pf, err := packages_service.OpenFileForDownloadByPackageNameAndVersion( ctx, &packages_service.PackageInfo{ Owner: ctx.Package.Owner, diff --git a/routers/api/packages/generic/generic.go b/routers/api/packages/generic/generic.go index 0b5daa7334..db7aeace50 100644 --- a/routers/api/packages/generic/generic.go +++ b/routers/api/packages/generic/generic.go @@ -31,7 +31,7 @@ func apiError(ctx *context.Context, status int, obj any) { // DownloadPackageFile serves the specific generic package. func DownloadPackageFile(ctx *context.Context) { - s, u, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + s, u, pf, err := packages_service.OpenFileForDownloadByPackageNameAndVersion( ctx, &packages_service.PackageInfo{ Owner: ctx.Package.Owner, diff --git a/routers/api/packages/goproxy/goproxy.go b/routers/api/packages/goproxy/goproxy.go index bde29df739..89ec86bce9 100644 --- a/routers/api/packages/goproxy/goproxy.go +++ b/routers/api/packages/goproxy/goproxy.go @@ -106,7 +106,7 @@ func DownloadPackageFile(ctx *context.Context) { return } - s, u, _, err := packages_service.GetPackageFileStream(ctx, pfs[0]) + s, u, _, err := packages_service.OpenFileForDownload(ctx, pfs[0]) if err != nil { if errors.Is(err, util.ErrNotExist) { apiError(ctx, http.StatusNotFound, err) diff --git a/routers/api/packages/helm/helm.go b/routers/api/packages/helm/helm.go index fb12daaa46..39c34f4da4 100644 --- a/routers/api/packages/helm/helm.go +++ b/routers/api/packages/helm/helm.go @@ -122,7 +122,7 @@ func DownloadPackageFile(ctx *context.Context) { return } - s, u, pf, err := packages_service.GetFileStreamByPackageVersion( + s, u, pf, err := packages_service.OpenFileForDownloadByPackageVersion( ctx, pvs[0], &packages_service.PackageFileInfo{ diff --git a/routers/api/packages/maven/maven.go b/routers/api/packages/maven/maven.go index 9089c2eccf..40a8ff8242 100644 --- a/routers/api/packages/maven/maven.go +++ b/routers/api/packages/maven/maven.go @@ -223,7 +223,7 @@ func servePackageFile(ctx *context.Context, params parameters, serveContent bool return } - s, u, _, err := packages_service.GetPackageBlobStream(ctx, pf, pb, nil) + s, u, _, err := packages_service.OpenBlobForDownload(ctx, pf, pb, nil) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/npm/npm.go b/routers/api/packages/npm/npm.go index 6ec46bcb36..1f09816d32 100644 --- a/routers/api/packages/npm/npm.go +++ b/routers/api/packages/npm/npm.go @@ -85,7 +85,7 @@ func DownloadPackageFile(ctx *context.Context) { packageVersion := ctx.PathParam("version") filename := ctx.PathParam("filename") - s, u, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + s, u, pf, err := packages_service.OpenFileForDownloadByPackageNameAndVersion( ctx, &packages_service.PackageInfo{ Owner: ctx.Package.Owner, @@ -132,7 +132,7 @@ func DownloadPackageFileByName(ctx *context.Context) { return } - s, u, pf, err := packages_service.GetFileStreamByPackageVersion( + s, u, pf, err := packages_service.OpenFileForDownloadByPackageVersion( ctx, pvs[0], &packages_service.PackageFileInfo{ diff --git a/routers/api/packages/nuget/api_v2.go b/routers/api/packages/nuget/api_v2.go index a726065ad0..801c60af13 100644 --- a/routers/api/packages/nuget/api_v2.go +++ b/routers/api/packages/nuget/api_v2.go @@ -246,21 +246,30 @@ type TypedValue[T any] struct { } type FeedEntryProperties struct { - Version string `xml:"d:Version"` - NormalizedVersion string `xml:"d:NormalizedVersion"` Authors string `xml:"d:Authors"` + Copyright string `xml:"d:Copyright,omitempty"` + Created TypedValue[time.Time] `xml:"d:Created"` Dependencies string `xml:"d:Dependencies"` Description string `xml:"d:Description"` - VersionDownloadCount TypedValue[int64] `xml:"d:VersionDownloadCount"` + DevelopmentDependency TypedValue[bool] `xml:"d:DevelopmentDependency"` DownloadCount TypedValue[int64] `xml:"d:DownloadCount"` - PackageSize TypedValue[int64] `xml:"d:PackageSize"` - Created TypedValue[time.Time] `xml:"d:Created"` + ID string `xml:"d:Id"` + IconURL string `xml:"d:IconUrl,omitempty"` + Language string `xml:"d:Language,omitempty"` LastUpdated TypedValue[time.Time] `xml:"d:LastUpdated"` - Published TypedValue[time.Time] `xml:"d:Published"` + LicenseURL string `xml:"d:LicenseUrl,omitempty"` + MinClientVersion string `xml:"d:MinClientVersion,omitempty"` + NormalizedVersion string `xml:"d:NormalizedVersion"` + Owners string `xml:"d:Owners,omitempty"` + PackageSize TypedValue[int64] `xml:"d:PackageSize"` ProjectURL string `xml:"d:ProjectUrl,omitempty"` + Published TypedValue[time.Time] `xml:"d:Published"` ReleaseNotes string `xml:"d:ReleaseNotes,omitempty"` RequireLicenseAcceptance TypedValue[bool] `xml:"d:RequireLicenseAcceptance"` - Title string `xml:"d:Title"` + Tags string `xml:"d:Tags,omitempty"` + Title string `xml:"d:Title,omitempty"` + Version string `xml:"d:Version"` + VersionDownloadCount TypedValue[int64] `xml:"d:VersionDownloadCount"` } type FeedEntry struct { @@ -353,21 +362,30 @@ func createEntry(l *linkBuilder, pd *packages_model.PackageDescriptor, withNames Author: metadata.Authors, Content: content, Properties: &FeedEntryProperties{ - Version: pd.Version.Version, - NormalizedVersion: pd.Version.Version, Authors: metadata.Authors, + Copyright: metadata.Copyright, + Created: createdValue, Dependencies: buildDependencyString(metadata), Description: metadata.Description, - VersionDownloadCount: TypedValue[int64]{Type: "Edm.Int64", Value: pd.Version.DownloadCount}, + DevelopmentDependency: TypedValue[bool]{Type: "Edm.Boolean", Value: metadata.DevelopmentDependency}, DownloadCount: TypedValue[int64]{Type: "Edm.Int64", Value: pd.Version.DownloadCount}, - PackageSize: TypedValue[int64]{Type: "Edm.Int64", Value: pd.CalculateBlobSize()}, - Created: createdValue, + ID: pd.Package.Name, + IconURL: metadata.IconURL, + Language: metadata.Language, LastUpdated: createdValue, - Published: createdValue, + LicenseURL: metadata.LicenseURL, + MinClientVersion: metadata.MinClientVersion, + NormalizedVersion: pd.Version.Version, + Owners: metadata.Owners, + PackageSize: TypedValue[int64]{Type: "Edm.Int64", Value: pd.CalculateBlobSize()}, ProjectURL: metadata.ProjectURL, + Published: createdValue, ReleaseNotes: metadata.ReleaseNotes, RequireLicenseAcceptance: TypedValue[bool]{Type: "Edm.Boolean", Value: metadata.RequireLicenseAcceptance}, - Title: pd.Package.Name, + Tags: metadata.Tags, + Title: metadata.Title, + Version: pd.Version.Version, + VersionDownloadCount: TypedValue[int64]{Type: "Edm.Int64", Value: pd.Version.DownloadCount}, }, } diff --git a/routers/api/packages/nuget/nuget.go b/routers/api/packages/nuget/nuget.go index fa5067a278..92d62d90b1 100644 --- a/routers/api/packages/nuget/nuget.go +++ b/routers/api/packages/nuget/nuget.go @@ -36,7 +36,7 @@ func apiError(ctx *context.Context, status int, obj any) { }) } -func xmlResponse(ctx *context.Context, status int, obj any) { //nolint:unparam +func xmlResponse(ctx *context.Context, status int, obj any) { //nolint:unparam // status is always StatusOK ctx.Resp.Header().Set("Content-Type", "application/atom+xml; charset=utf-8") ctx.Resp.WriteHeader(status) if _, err := ctx.Resp.Write([]byte(xml.Header)); err != nil { @@ -405,7 +405,7 @@ func DownloadPackageFile(ctx *context.Context) { packageVersion := ctx.PathParam("version") filename := ctx.PathParam("filename") - s, u, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + s, u, pf, err := packages_service.OpenFileForDownloadByPackageNameAndVersion( ctx, &packages_service.PackageInfo{ Owner: ctx.Package.Owner, @@ -669,7 +669,7 @@ func DownloadSymbolFile(ctx *context.Context) { return } - s, u, pf, err := packages_service.GetPackageFileStream(ctx, pfs[0]) + s, u, pf, err := packages_service.OpenFileForDownload(ctx, pfs[0]) if err != nil { if errors.Is(err, packages_model.ErrPackageNotExist) || errors.Is(err, packages_model.ErrPackageFileNotExist) { apiError(ctx, http.StatusNotFound, err) diff --git a/routers/api/packages/pub/pub.go b/routers/api/packages/pub/pub.go index e7b07aefd0..4bd36e94b6 100644 --- a/routers/api/packages/pub/pub.go +++ b/routers/api/packages/pub/pub.go @@ -274,7 +274,7 @@ func DownloadPackageFile(ctx *context.Context) { pf := pd.Files[0].File - s, u, _, err := packages_service.GetPackageFileStream(ctx, pf) + s, u, _, err := packages_service.OpenFileForDownload(ctx, pf) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/pypi/pypi.go b/routers/api/packages/pypi/pypi.go index 199f4e7478..9b5ae6c89d 100644 --- a/routers/api/packages/pypi/pypi.go +++ b/routers/api/packages/pypi/pypi.go @@ -82,7 +82,7 @@ func DownloadPackageFile(ctx *context.Context) { packageVersion := ctx.PathParam("version") filename := ctx.PathParam("filename") - s, u, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + s, u, pf, err := packages_service.OpenFileForDownloadByPackageNameAndVersion( ctx, &packages_service.PackageInfo{ Owner: ctx.Package.Owner, diff --git a/routers/api/packages/rpm/rpm.go b/routers/api/packages/rpm/rpm.go index a00a61c079..938c35341d 100644 --- a/routers/api/packages/rpm/rpm.go +++ b/routers/api/packages/rpm/rpm.go @@ -96,7 +96,7 @@ func GetRepositoryFile(ctx *context.Context) { return } - s, u, pf, err := packages_service.GetFileStreamByPackageVersion( + s, u, pf, err := packages_service.OpenFileForDownloadByPackageVersion( ctx, pv, &packages_service.PackageFileInfo{ @@ -220,7 +220,7 @@ func DownloadPackageFile(ctx *context.Context) { name := ctx.PathParam("name") version := ctx.PathParam("version") - s, u, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + s, u, pf, err := packages_service.OpenFileForDownloadByPackageNameAndVersion( ctx, &packages_service.PackageInfo{ Owner: ctx.Package.Owner, diff --git a/routers/api/packages/rubygems/rubygems.go b/routers/api/packages/rubygems/rubygems.go index de8c7ef3ed..774d5520fd 100644 --- a/routers/api/packages/rubygems/rubygems.go +++ b/routers/api/packages/rubygems/rubygems.go @@ -14,6 +14,7 @@ import ( "strings" packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/optional" packages_module "code.gitea.io/gitea/modules/packages" rubygems_module "code.gitea.io/gitea/modules/packages/rubygems" @@ -177,7 +178,7 @@ func DownloadPackageFile(ctx *context.Context) { return } - s, u, pf, err := packages_service.GetFileStreamByPackageVersion( + s, u, pf, err := packages_service.OpenFileForDownloadByPackageVersion( ctx, pvs[0], &packages_service.PackageFileInfo{ @@ -309,7 +310,7 @@ func GetPackageInfo(ctx *context.Context) { apiError(ctx, http.StatusNotFound, nil) return } - infoContent, err := makePackageInfo(ctx, versions) + infoContent, err := makePackageInfo(ctx, versions, cache.NewEphemeralCache()) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -317,7 +318,7 @@ func GetPackageInfo(ctx *context.Context) { ctx.PlainText(http.StatusOK, infoContent) } -// GetAllPackagesVersions returns a custom text based format containing information about all versions of all rubygems. +// GetAllPackagesVersions returns a custom text-based format containing information about all versions of all rubygems. // ref: https://guides.rubygems.org/rubygems-org-compact-index-api/ func GetAllPackagesVersions(ctx *context.Context) { packages, err := packages_model.GetPackagesByType(ctx, ctx.Package.Owner.ID, packages_model.TypeRubyGems) @@ -326,6 +327,7 @@ func GetAllPackagesVersions(ctx *context.Context) { return } + ephemeralCache := cache.NewEphemeralCache() out := &strings.Builder{} out.WriteString("---\n") for _, pkg := range packages { @@ -338,7 +340,7 @@ func GetAllPackagesVersions(ctx *context.Context) { continue } - info, err := makePackageInfo(ctx, versions) + info, err := makePackageInfo(ctx, versions, ephemeralCache) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -348,7 +350,14 @@ func GetAllPackagesVersions(ctx *context.Context) { _, _ = fmt.Fprintf(out, "%s ", pkg.Name) for i, v := range versions { sep := util.Iif(i == len(versions)-1, "", ",") - _, _ = fmt.Fprintf(out, "%s%s", v.Version, sep) + pd, err := packages_model.GetPackageDescriptorWithCache(ctx, v, ephemeralCache) + if errors.Is(err, util.ErrNotExist) { + continue + } else if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + writePackageVersionForList(pd.Metadata, v.Version, sep, out) } _, _ = fmt.Fprintf(out, " %x\n", md5.Sum([]byte(info))) } @@ -356,6 +365,16 @@ func GetAllPackagesVersions(ctx *context.Context) { ctx.PlainText(http.StatusOK, out.String()) } +func writePackageVersionForList(metadata any, version, sep string, out *strings.Builder) { + if metadata, _ := metadata.(*rubygems_module.Metadata); metadata != nil && metadata.Platform != "" && metadata.Platform != "ruby" { + // VERSION_PLATFORM (see comment above in GetAllPackagesVersions) + _, _ = fmt.Fprintf(out, "%s_%s%s", version, metadata.Platform, sep) + } else { + // VERSION only + _, _ = fmt.Fprintf(out, "%s%s", version, sep) + } +} + func writePackageVersionRequirements(prefix string, reqs []rubygems_module.VersionRequirement, out *strings.Builder) { out.WriteString(prefix) if len(reqs) == 0 { @@ -367,11 +386,21 @@ func writePackageVersionRequirements(prefix string, reqs []rubygems_module.Versi } } -func makePackageVersionDependency(ctx *context.Context, version *packages_model.PackageVersion) (string, error) { +func writePackageVersionForDependency(version, platform string, out *strings.Builder) { + if platform != "" && platform != "ruby" { + // VERSION-PLATFORM (see comment below in makePackageVersionDependency) + _, _ = fmt.Fprintf(out, "%s-%s ", version, platform) + } else { + // VERSION only + _, _ = fmt.Fprintf(out, "%s ", version) + } +} + +func makePackageVersionDependency(ctx *context.Context, version *packages_model.PackageVersion, c *cache.EphemeralCache) (string, error) { // format: VERSION[-PLATFORM] [DEPENDENCY[,DEPENDENCY,...]]|REQUIREMENT[,REQUIREMENT,...] // DEPENDENCY: GEM:CONSTRAINT[&CONSTRAINT] // REQUIREMENT: KEY:VALUE (always contains "checksum") - pd, err := packages_model.GetPackageDescriptor(ctx, version) + pd, err := packages_model.GetPackageDescriptorWithCache(ctx, version, c) if err != nil { return "", err } @@ -388,8 +417,7 @@ func makePackageVersionDependency(ctx *context.Context, version *packages_model. } buf := &strings.Builder{} - buf.WriteString(version.Version) - buf.WriteByte(' ') + writePackageVersionForDependency(version.Version, metadata.Platform, buf) for i, dep := range metadata.RuntimeDependencies { sep := util.Iif(i == 0, "", ",") writePackageVersionRequirements(fmt.Sprintf("%s%s:", sep, dep.Name), dep.Version, buf) @@ -404,10 +432,10 @@ func makePackageVersionDependency(ctx *context.Context, version *packages_model. return buf.String(), nil } -func makePackageInfo(ctx *context.Context, versions []*packages_model.PackageVersion) (string, error) { +func makePackageInfo(ctx *context.Context, versions []*packages_model.PackageVersion, c *cache.EphemeralCache) (string, error) { ret := "---\n" for _, v := range versions { - dep, err := makePackageVersionDependency(ctx, v) + dep, err := makePackageVersionDependency(ctx, v, c) if err != nil { return "", err } diff --git a/routers/api/packages/rubygems/rubygems_test.go b/routers/api/packages/rubygems/rubygems_test.go new file mode 100644 index 0000000000..a07e12a7d3 --- /dev/null +++ b/routers/api/packages/rubygems/rubygems_test.go @@ -0,0 +1,41 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package rubygems + +import ( + "strings" + "testing" + + rubygems_module "code.gitea.io/gitea/modules/packages/rubygems" + + "github.com/stretchr/testify/assert" +) + +func TestWritePackageVersion(t *testing.T) { + buf := &strings.Builder{} + + writePackageVersionForList(nil, "1.0", " ", buf) + assert.Equal(t, "1.0 ", buf.String()) + buf.Reset() + + writePackageVersionForList(&rubygems_module.Metadata{Platform: "ruby"}, "1.0", " ", buf) + assert.Equal(t, "1.0 ", buf.String()) + buf.Reset() + + writePackageVersionForList(&rubygems_module.Metadata{Platform: "linux"}, "1.0", " ", buf) + assert.Equal(t, "1.0_linux ", buf.String()) + buf.Reset() + + writePackageVersionForDependency("1.0", "", buf) + assert.Equal(t, "1.0 ", buf.String()) + buf.Reset() + + writePackageVersionForDependency("1.0", "ruby", buf) + assert.Equal(t, "1.0 ", buf.String()) + buf.Reset() + + writePackageVersionForDependency("1.0", "os", buf) + assert.Equal(t, "1.0-os ", buf.String()) + buf.Reset() +} diff --git a/routers/api/packages/swift/swift.go b/routers/api/packages/swift/swift.go index 47439c4c3b..bf542f33a7 100644 --- a/routers/api/packages/swift/swift.go +++ b/routers/api/packages/swift/swift.go @@ -429,7 +429,7 @@ func DownloadPackageFile(ctx *context.Context) { pf := pd.Files[0].File - s, u, _, err := packages_service.GetPackageFileStream(ctx, pf) + s, u, _, err := packages_service.OpenFileForDownload(ctx, pf) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/vagrant/vagrant.go b/routers/api/packages/vagrant/vagrant.go index 3afaa5de1f..9eb67e5397 100644 --- a/routers/api/packages/vagrant/vagrant.go +++ b/routers/api/packages/vagrant/vagrant.go @@ -218,7 +218,7 @@ func UploadPackageFile(ctx *context.Context) { } func DownloadPackageFile(ctx *context.Context) { - s, u, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + s, u, pf, err := packages_service.OpenFileForDownloadByPackageNameAndVersion( ctx, &packages_service.PackageInfo{ Owner: ctx.Package.Owner, |