diff options
Diffstat (limited to 'routers/api/packages')
22 files changed, 5237 insertions, 0 deletions
diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go new file mode 100644 index 0000000000..f0251b95eb --- /dev/null +++ b/routers/api/packages/api.go @@ -0,0 +1,397 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "net/http" + "regexp" + "strings" + + "code.gitea.io/gitea/models/perm" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/web" + "code.gitea.io/gitea/routers/api/packages/composer" + "code.gitea.io/gitea/routers/api/packages/conan" + "code.gitea.io/gitea/routers/api/packages/container" + "code.gitea.io/gitea/routers/api/packages/generic" + "code.gitea.io/gitea/routers/api/packages/maven" + "code.gitea.io/gitea/routers/api/packages/npm" + "code.gitea.io/gitea/routers/api/packages/nuget" + "code.gitea.io/gitea/routers/api/packages/pypi" + "code.gitea.io/gitea/routers/api/packages/rubygems" + "code.gitea.io/gitea/services/auth" + context_service "code.gitea.io/gitea/services/context" +) + +func reqPackageAccess(accessMode perm.AccessMode) func(ctx *context.Context) { + return func(ctx *context.Context) { + if ctx.Package.AccessMode < accessMode && !ctx.IsUserSiteAdmin() { + ctx.Resp.Header().Set("WWW-Authenticate", `Basic realm="Gitea Package API"`) + ctx.Error(http.StatusUnauthorized, "reqPackageAccess", "user should have specific permission or be a site admin") + return + } + } +} + +func Routes() *web.Route { + r := web.NewRoute() + + r.Use(context.PackageContexter()) + + authMethods := []auth.Method{ + &auth.OAuth2{}, + &auth.Basic{}, + &conan.Auth{}, + } + if setting.Service.EnableReverseProxyAuth { + authMethods = append(authMethods, &auth.ReverseProxy{}) + } + + authGroup := auth.NewGroup(authMethods...) + r.Use(func(ctx *context.Context) { + ctx.Doer = authGroup.Verify(ctx.Req, ctx.Resp, ctx, ctx.Session) + }) + + r.Group("/{username}", func() { + r.Group("/composer", func() { + r.Get("/packages.json", composer.ServiceIndex) + r.Get("/search.json", composer.SearchPackages) + r.Get("/list.json", composer.EnumeratePackages) + r.Get("/p2/{vendorname}/{projectname}~dev.json", composer.PackageMetadata) + r.Get("/p2/{vendorname}/{projectname}.json", composer.PackageMetadata) + r.Get("/files/{package}/{version}/{filename}", composer.DownloadPackageFile) + r.Put("", reqPackageAccess(perm.AccessModeWrite), composer.UploadPackage) + }) + r.Group("/conan", func() { + r.Group("/v1", func() { + r.Get("/ping", conan.Ping) + r.Group("/users", func() { + r.Get("/authenticate", conan.Authenticate) + r.Get("/check_credentials", conan.CheckCredentials) + }) + r.Group("/conans", func() { + r.Get("/search", conan.SearchRecipes) + r.Group("/{name}/{version}/{user}/{channel}", func() { + r.Get("", conan.RecipeSnapshot) + r.Delete("", reqPackageAccess(perm.AccessModeWrite), conan.DeleteRecipeV1) + r.Get("/search", conan.SearchPackagesV1) + r.Get("/digest", conan.RecipeDownloadURLs) + r.Post("/upload_urls", reqPackageAccess(perm.AccessModeWrite), conan.RecipeUploadURLs) + r.Get("/download_urls", conan.RecipeDownloadURLs) + r.Group("/packages", func() { + r.Post("/delete", reqPackageAccess(perm.AccessModeWrite), conan.DeletePackageV1) + r.Group("/{package_reference}", func() { + r.Get("", conan.PackageSnapshot) + r.Get("/digest", conan.PackageDownloadURLs) + r.Post("/upload_urls", reqPackageAccess(perm.AccessModeWrite), conan.PackageUploadURLs) + r.Get("/download_urls", conan.PackageDownloadURLs) + }) + }) + }, conan.ExtractPathParameters) + }) + r.Group("/files/{name}/{version}/{user}/{channel}/{recipe_revision}", func() { + r.Group("/recipe/{filename}", func() { + r.Get("", conan.DownloadRecipeFile) + r.Put("", reqPackageAccess(perm.AccessModeWrite), conan.UploadRecipeFile) + }) + r.Group("/package/{package_reference}/{package_revision}/{filename}", func() { + r.Get("", conan.DownloadPackageFile) + r.Put("", reqPackageAccess(perm.AccessModeWrite), conan.UploadPackageFile) + }) + }, conan.ExtractPathParameters) + }) + r.Group("/v2", func() { + r.Get("/ping", conan.Ping) + r.Group("/users", func() { + r.Get("/authenticate", conan.Authenticate) + r.Get("/check_credentials", conan.CheckCredentials) + }) + r.Group("/conans", func() { + r.Get("/search", conan.SearchRecipes) + r.Group("/{name}/{version}/{user}/{channel}", func() { + r.Delete("", reqPackageAccess(perm.AccessModeWrite), conan.DeleteRecipeV2) + r.Get("/search", conan.SearchPackagesV2) + r.Get("/latest", conan.LatestRecipeRevision) + r.Group("/revisions", func() { + r.Get("", conan.ListRecipeRevisions) + r.Group("/{recipe_revision}", func() { + r.Delete("", reqPackageAccess(perm.AccessModeWrite), conan.DeleteRecipeV2) + r.Get("/search", conan.SearchPackagesV2) + r.Group("/files", func() { + r.Get("", conan.ListRecipeRevisionFiles) + r.Group("/{filename}", func() { + r.Get("", conan.DownloadRecipeFile) + r.Put("", reqPackageAccess(perm.AccessModeWrite), conan.UploadRecipeFile) + }) + }) + r.Group("/packages", func() { + r.Delete("", reqPackageAccess(perm.AccessModeWrite), conan.DeletePackageV2) + r.Group("/{package_reference}", func() { + r.Delete("", reqPackageAccess(perm.AccessModeWrite), conan.DeletePackageV2) + r.Get("/latest", conan.LatestPackageRevision) + r.Group("/revisions", func() { + r.Get("", conan.ListPackageRevisions) + r.Group("/{package_revision}", func() { + r.Delete("", reqPackageAccess(perm.AccessModeWrite), conan.DeletePackageV2) + r.Group("/files", func() { + r.Get("", conan.ListPackageRevisionFiles) + r.Group("/{filename}", func() { + r.Get("", conan.DownloadPackageFile) + r.Put("", reqPackageAccess(perm.AccessModeWrite), conan.UploadPackageFile) + }) + }) + }) + }) + }) + }) + }) + }) + }, conan.ExtractPathParameters) + }) + }) + }) + r.Group("/generic", func() { + r.Group("/{packagename}/{packageversion}/{filename}", func() { + r.Get("", generic.DownloadPackageFile) + r.Group("", func() { + r.Put("", generic.UploadPackage) + r.Delete("", generic.DeletePackage) + }, reqPackageAccess(perm.AccessModeWrite)) + }) + }) + r.Group("/maven", func() { + r.Put("/*", reqPackageAccess(perm.AccessModeWrite), maven.UploadPackageFile) + r.Get("/*", maven.DownloadPackageFile) + }) + r.Group("/nuget", func() { + r.Get("/index.json", nuget.ServiceIndex) + r.Get("/query", nuget.SearchService) + r.Group("/registration/{id}", func() { + r.Get("/index.json", nuget.RegistrationIndex) + r.Get("/{version}", nuget.RegistrationLeaf) + }) + r.Group("/package/{id}", func() { + r.Get("/index.json", nuget.EnumeratePackageVersions) + r.Get("/{version}/{filename}", nuget.DownloadPackageFile) + }) + r.Group("", func() { + r.Put("/", nuget.UploadPackage) + r.Put("/symbolpackage", nuget.UploadSymbolPackage) + r.Delete("/{id}/{version}", nuget.DeletePackage) + }, reqPackageAccess(perm.AccessModeWrite)) + r.Get("/symbols/{filename}/{guid:[0-9a-f]{32}}FFFFFFFF/{filename2}", nuget.DownloadSymbolFile) + }) + r.Group("/npm", func() { + r.Group("/@{scope}/{id}", func() { + r.Get("", npm.PackageMetadata) + r.Put("", reqPackageAccess(perm.AccessModeWrite), npm.UploadPackage) + r.Get("/-/{version}/{filename}", npm.DownloadPackageFile) + }) + r.Group("/{id}", func() { + r.Get("", npm.PackageMetadata) + r.Put("", reqPackageAccess(perm.AccessModeWrite), npm.UploadPackage) + r.Get("/-/{version}/{filename}", npm.DownloadPackageFile) + }) + r.Group("/-/package/@{scope}/{id}/dist-tags", func() { + r.Get("", npm.ListPackageTags) + r.Group("/{tag}", func() { + r.Put("", npm.AddPackageTag) + r.Delete("", npm.DeletePackageTag) + }, reqPackageAccess(perm.AccessModeWrite)) + }) + r.Group("/-/package/{id}/dist-tags", func() { + r.Get("", npm.ListPackageTags) + r.Group("/{tag}", func() { + r.Put("", npm.AddPackageTag) + r.Delete("", npm.DeletePackageTag) + }, reqPackageAccess(perm.AccessModeWrite)) + }) + }) + r.Group("/pypi", func() { + r.Post("/", reqPackageAccess(perm.AccessModeWrite), pypi.UploadPackageFile) + r.Get("/files/{id}/{version}/{filename}", pypi.DownloadPackageFile) + r.Get("/simple/{id}", pypi.PackageMetadata) + }) + r.Group("/rubygems", func() { + r.Get("/specs.4.8.gz", rubygems.EnumeratePackages) + r.Get("/latest_specs.4.8.gz", rubygems.EnumeratePackagesLatest) + r.Get("/prerelease_specs.4.8.gz", rubygems.EnumeratePackagesPreRelease) + r.Get("/quick/Marshal.4.8/{filename}", rubygems.ServePackageSpecification) + r.Get("/gems/{filename}", rubygems.DownloadPackageFile) + r.Group("/api/v1/gems", func() { + r.Post("/", rubygems.UploadPackageFile) + r.Delete("/yank", rubygems.DeletePackage) + }, reqPackageAccess(perm.AccessModeWrite)) + }) + }, context_service.UserAssignmentWeb(), context.PackageAssignment(), reqPackageAccess(perm.AccessModeRead)) + + return r +} + +func ContainerRoutes() *web.Route { + r := web.NewRoute() + + r.Use(context.PackageContexter()) + + authMethods := []auth.Method{ + &auth.Basic{}, + &container.Auth{}, + } + if setting.Service.EnableReverseProxyAuth { + authMethods = append(authMethods, &auth.ReverseProxy{}) + } + + authGroup := auth.NewGroup(authMethods...) + r.Use(func(ctx *context.Context) { + ctx.Doer = authGroup.Verify(ctx.Req, ctx.Resp, ctx, ctx.Session) + }) + + r.Get("", container.ReqContainerAccess, container.DetermineSupport) + r.Get("/token", container.Authenticate) + r.Group("/{username}", func() { + r.Group("/{image}", func() { + r.Group("/blobs/uploads", func() { + r.Post("", container.InitiateUploadBlob) + r.Group("/{uuid}", func() { + r.Patch("", container.UploadBlob) + r.Put("", container.EndUploadBlob) + }) + }, reqPackageAccess(perm.AccessModeWrite)) + r.Group("/blobs/{digest}", func() { + r.Head("", container.HeadBlob) + r.Get("", container.GetBlob) + r.Delete("", reqPackageAccess(perm.AccessModeWrite), container.DeleteBlob) + }) + r.Group("/manifests/{reference}", func() { + r.Put("", reqPackageAccess(perm.AccessModeWrite), container.UploadManifest) + r.Head("", container.HeadManifest) + r.Get("", container.GetManifest) + r.Delete("", reqPackageAccess(perm.AccessModeWrite), container.DeleteManifest) + }) + r.Get("/tags/list", container.GetTagList) + }, container.VerifyImageName) + + var ( + blobsUploadsPattern = regexp.MustCompile(`\A(.+)/blobs/uploads/([a-zA-Z0-9-_.=]+)\z`) + blobsPattern = regexp.MustCompile(`\A(.+)/blobs/([^/]+)\z`) + manifestsPattern = regexp.MustCompile(`\A(.+)/manifests/([^/]+)\z`) + ) + + // Manual mapping of routes because {image} can contain slashes which chi does not support + r.Route("/*", "HEAD,GET,POST,PUT,PATCH,DELETE", func(ctx *context.Context) { + path := ctx.Params("*") + isHead := ctx.Req.Method == "HEAD" + isGet := ctx.Req.Method == "GET" + isPost := ctx.Req.Method == "POST" + isPut := ctx.Req.Method == "PUT" + isPatch := ctx.Req.Method == "PATCH" + isDelete := ctx.Req.Method == "DELETE" + + if isPost && strings.HasSuffix(path, "/blobs/uploads") { + reqPackageAccess(perm.AccessModeWrite)(ctx) + if ctx.Written() { + return + } + + ctx.SetParams("image", path[:len(path)-14]) + container.VerifyImageName(ctx) + if ctx.Written() { + return + } + + container.InitiateUploadBlob(ctx) + return + } + if isGet && strings.HasSuffix(path, "/tags/list") { + ctx.SetParams("image", path[:len(path)-10]) + container.VerifyImageName(ctx) + if ctx.Written() { + return + } + + container.GetTagList(ctx) + return + } + + m := blobsUploadsPattern.FindStringSubmatch(path) + if len(m) == 3 && (isPut || isPatch) { + reqPackageAccess(perm.AccessModeWrite)(ctx) + if ctx.Written() { + return + } + + ctx.SetParams("image", m[1]) + container.VerifyImageName(ctx) + if ctx.Written() { + return + } + + ctx.SetParams("uuid", m[2]) + + if isPatch { + container.UploadBlob(ctx) + } else { + container.EndUploadBlob(ctx) + } + return + } + m = blobsPattern.FindStringSubmatch(path) + if len(m) == 3 && (isHead || isGet || isDelete) { + ctx.SetParams("image", m[1]) + container.VerifyImageName(ctx) + if ctx.Written() { + return + } + + ctx.SetParams("digest", m[2]) + + if isHead { + container.HeadBlob(ctx) + } else if isGet { + container.GetBlob(ctx) + } else { + reqPackageAccess(perm.AccessModeWrite)(ctx) + if ctx.Written() { + return + } + container.DeleteBlob(ctx) + } + return + } + m = manifestsPattern.FindStringSubmatch(path) + if len(m) == 3 && (isHead || isGet || isPut || isDelete) { + ctx.SetParams("image", m[1]) + container.VerifyImageName(ctx) + if ctx.Written() { + return + } + + ctx.SetParams("reference", m[2]) + + if isHead { + container.HeadManifest(ctx) + } else if isGet { + container.GetManifest(ctx) + } else { + reqPackageAccess(perm.AccessModeWrite)(ctx) + if ctx.Written() { + return + } + if isPut { + container.UploadManifest(ctx) + } else { + container.DeleteManifest(ctx) + } + } + return + } + + ctx.Status(http.StatusNotFound) + }) + }, container.ReqContainerAccess, context_service.UserAssignmentWeb(), context.PackageAssignment(), reqPackageAccess(perm.AccessModeRead)) + + return r +} diff --git a/routers/api/packages/composer/api.go b/routers/api/packages/composer/api.go new file mode 100644 index 0000000000..d8f67d130c --- /dev/null +++ b/routers/api/packages/composer/api.go @@ -0,0 +1,118 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package composer + +import ( + "fmt" + "net/url" + "time" + + packages_model "code.gitea.io/gitea/models/packages" + composer_module "code.gitea.io/gitea/modules/packages/composer" +) + +// ServiceIndexResponse contains registry endpoints +type ServiceIndexResponse struct { + SearchTemplate string `json:"search"` + MetadataTemplate string `json:"metadata-url"` + PackageList string `json:"list"` +} + +func createServiceIndexResponse(registryURL string) *ServiceIndexResponse { + return &ServiceIndexResponse{ + SearchTemplate: registryURL + "/search.json?q=%query%&type=%type%", + MetadataTemplate: registryURL + "/p2/%package%.json", + PackageList: registryURL + "/list.json", + } +} + +// SearchResultResponse contains search results +type SearchResultResponse struct { + Total int64 `json:"total"` + Results []*SearchResult `json:"results"` + NextLink string `json:"next,omitempty"` +} + +// SearchResult contains a search result +type SearchResult struct { + Name string `json:"name"` + Description string `json:"description"` + Downloads int64 `json:"downloads"` +} + +func createSearchResultResponse(total int64, pds []*packages_model.PackageDescriptor, nextLink string) *SearchResultResponse { + results := make([]*SearchResult, 0, len(pds)) + + for _, pd := range pds { + results = append(results, &SearchResult{ + Name: pd.Package.Name, + Description: pd.Metadata.(*composer_module.Metadata).Description, + Downloads: pd.Version.DownloadCount, + }) + } + + return &SearchResultResponse{ + Total: total, + Results: results, + NextLink: nextLink, + } +} + +// PackageMetadataResponse contains packages metadata +type PackageMetadataResponse struct { + Minified string `json:"minified"` + Packages map[string][]*PackageVersionMetadata `json:"packages"` +} + +// PackageVersionMetadata contains package metadata +type PackageVersionMetadata struct { + *composer_module.Metadata + Name string `json:"name"` + Version string `json:"version"` + Type string `json:"type"` + Created time.Time `json:"time"` + Dist Dist `json:"dist"` +} + +// Dist contains package download informations +type Dist struct { + Type string `json:"type"` + URL string `json:"url"` + Checksum string `json:"shasum"` +} + +func createPackageMetadataResponse(registryURL string, pds []*packages_model.PackageDescriptor) *PackageMetadataResponse { + versions := make([]*PackageVersionMetadata, 0, len(pds)) + + for _, pd := range pds { + packageType := "" + for _, pvp := range pd.Properties { + if pvp.Name == composer_module.TypeProperty { + packageType = pvp.Value + break + } + } + + versions = append(versions, &PackageVersionMetadata{ + Name: pd.Package.Name, + Version: pd.Version.Version, + Type: packageType, + Created: time.Unix(int64(pd.Version.CreatedUnix), 0), + Metadata: pd.Metadata.(*composer_module.Metadata), + Dist: Dist{ + Type: "zip", + URL: fmt.Sprintf("%s/files/%s/%s/%s", registryURL, url.PathEscape(pd.Package.LowerName), url.PathEscape(pd.Version.LowerVersion), url.PathEscape(pd.Files[0].File.LowerName)), + Checksum: pd.Files[0].Blob.HashSHA1, + }, + }) + } + + return &PackageMetadataResponse{ + Minified: "composer/2.0", + Packages: map[string][]*PackageVersionMetadata{ + pds[0].Package.Name: versions, + }, + } +} diff --git a/routers/api/packages/composer/composer.go b/routers/api/packages/composer/composer.go new file mode 100644 index 0000000000..22a452325e --- /dev/null +++ b/routers/api/packages/composer/composer.go @@ -0,0 +1,250 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package composer + +import ( + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/convert" + packages_module "code.gitea.io/gitea/modules/packages" + composer_module "code.gitea.io/gitea/modules/packages/composer" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/routers/api/packages/helper" + packages_service "code.gitea.io/gitea/services/packages" + + "github.com/hashicorp/go-version" +) + +func apiError(ctx *context.Context, status int, obj interface{}) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { + type Error struct { + Status int `json:"status"` + Message string `json:"message"` + } + ctx.JSON(status, struct { + Errors []Error `json:"errors"` + }{ + Errors: []Error{ + {Status: status, Message: message}, + }, + }) + }) +} + +// ServiceIndex displays registry endpoints +func ServiceIndex(ctx *context.Context) { + resp := createServiceIndexResponse(setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/composer") + + ctx.JSON(http.StatusOK, resp) +} + +// SearchPackages searches packages, only "q" is supported +// https://packagist.org/apidoc#search-packages +func SearchPackages(ctx *context.Context) { + page := ctx.FormInt("page") + if page < 1 { + page = 1 + } + perPage := ctx.FormInt("per_page") + paginator := db.ListOptions{ + Page: page, + PageSize: convert.ToCorrectPageSize(perPage), + } + + opts := &packages_model.PackageSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + Type: string(packages_model.TypeComposer), + QueryName: ctx.FormTrim("q"), + Paginator: &paginator, + } + if ctx.FormTrim("type") != "" { + opts.Properties = map[string]string{ + composer_module.TypeProperty: ctx.FormTrim("type"), + } + } + + pvs, total, err := packages_model.SearchLatestVersions(ctx, opts) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + nextLink := "" + if len(pvs) == paginator.PageSize { + u, err := url.Parse(setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/composer/search.json") + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + q := u.Query() + q.Set("q", ctx.FormTrim("q")) + q.Set("type", ctx.FormTrim("type")) + q.Set("page", strconv.Itoa(page+1)) + if perPage != 0 { + q.Set("per_page", strconv.Itoa(perPage)) + } + u.RawQuery = q.Encode() + + nextLink = u.String() + } + + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + resp := createSearchResultResponse(total, pds, nextLink) + + ctx.JSON(http.StatusOK, resp) +} + +// EnumeratePackages lists all package names +// https://packagist.org/apidoc#list-packages +func EnumeratePackages(ctx *context.Context) { + ps, err := packages_model.GetPackagesByType(db.DefaultContext, ctx.Package.Owner.ID, packages_model.TypeComposer) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + names := make([]string, 0, len(ps)) + for _, p := range ps { + names = append(names, p.Name) + } + + ctx.JSON(http.StatusOK, map[string][]string{ + "packageNames": names, + }) +} + +// PackageMetadata returns the metadata for a single package +// https://packagist.org/apidoc#get-package-data +func PackageMetadata(ctx *context.Context) { + vendorName := ctx.Params("vendorname") + projectName := ctx.Params("projectname") + + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeComposer, vendorName+"/"+projectName) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pvs) == 0 { + apiError(ctx, http.StatusNotFound, packages_model.ErrPackageNotExist) + return + } + + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + resp := createPackageMetadataResponse( + setting.AppURL+"api/packages/"+ctx.Package.Owner.Name+"/composer", + pds, + ) + + ctx.JSON(http.StatusOK, resp) +} + +// DownloadPackageFile serves the content of a package +func DownloadPackageFile(ctx *context.Context) { + s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + ctx, + &packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeComposer, + Name: ctx.Params("package"), + Version: ctx.Params("version"), + }, + &packages_service.PackageFileInfo{ + Filename: ctx.Params("filename"), + }, + ) + if err != nil { + if err == packages_model.ErrPackageNotExist || err == packages_model.ErrPackageFileNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer s.Close() + + ctx.ServeStream(s, pf.Name) +} + +// UploadPackage creates a new package +func UploadPackage(ctx *context.Context) { + buf, err := packages_module.CreateHashedBufferFromReader(ctx.Req.Body, 32*1024*1024) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + cp, err := composer_module.ParsePackage(buf, buf.Size()) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if cp.Version == "" { + v, err := version.NewVersion(ctx.FormTrim("version")) + if err != nil { + apiError(ctx, http.StatusBadRequest, composer_module.ErrInvalidVersion) + return + } + cp.Version = v.String() + } + + _, _, err = packages_service.CreatePackageAndAddFile( + &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeComposer, + Name: cp.Name, + Version: cp.Version, + }, + SemverCompatible: true, + Creator: ctx.Doer, + Metadata: cp.Metadata, + Properties: map[string]string{ + composer_module.TypeProperty: cp.Type, + }, + }, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: strings.ToLower(fmt.Sprintf("%s.%s.zip", strings.ReplaceAll(cp.Name, "/", "-"), cp.Version)), + }, + Data: buf, + IsLead: true, + }, + ) + if err != nil { + if err == packages_model.ErrDuplicatePackageVersion { + apiError(ctx, http.StatusBadRequest, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.Status(http.StatusCreated) +} diff --git a/routers/api/packages/conan/auth.go b/routers/api/packages/conan/auth.go new file mode 100644 index 0000000000..00855a97a4 --- /dev/null +++ b/routers/api/packages/conan/auth.go @@ -0,0 +1,41 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package conan + +import ( + "net/http" + + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/services/auth" + "code.gitea.io/gitea/services/packages" +) + +type Auth struct{} + +func (a *Auth) Name() string { + return "conan" +} + +// Verify extracts the user from the Bearer token +func (a *Auth) Verify(req *http.Request, w http.ResponseWriter, store auth.DataStore, sess auth.SessionStore) *user_model.User { + uid, err := packages.ParseAuthorizationToken(req) + if err != nil { + log.Trace("ParseAuthorizationToken: %v", err) + return nil + } + + if uid == 0 { + return nil + } + + u, err := user_model.GetUserByID(uid) + if err != nil { + log.Error("GetUserByID: %v", err) + return nil + } + + return u +} diff --git a/routers/api/packages/conan/conan.go b/routers/api/packages/conan/conan.go new file mode 100644 index 0000000000..0a27f18fd1 --- /dev/null +++ b/routers/api/packages/conan/conan.go @@ -0,0 +1,818 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package conan + +import ( + "fmt" + "io" + "net/http" + "strings" + "time" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + conan_model "code.gitea.io/gitea/models/packages/conan" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/notification" + packages_module "code.gitea.io/gitea/modules/packages" + conan_module "code.gitea.io/gitea/modules/packages/conan" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/routers/api/packages/helper" + packages_service "code.gitea.io/gitea/services/packages" +) + +const ( + conanfileFile = "conanfile.py" + conaninfoFile = "conaninfo.txt" + + recipeReferenceKey = "RecipeReference" + packageReferenceKey = "PackageReference" +) + +type stringSet map[string]struct{} + +var ( + recipeFileList = stringSet{ + conanfileFile: struct{}{}, + "conanmanifest.txt": struct{}{}, + "conan_sources.tgz": struct{}{}, + "conan_export.tgz": struct{}{}, + } + packageFileList = stringSet{ + conaninfoFile: struct{}{}, + "conanmanifest.txt": struct{}{}, + "conan_package.tgz": struct{}{}, + } +) + +func jsonResponse(ctx *context.Context, status int, obj interface{}) { + // https://github.com/conan-io/conan/issues/6613 + ctx.Resp.Header().Set("Content-Type", "application/json") + ctx.Status(status) + if err := json.NewEncoder(ctx.Resp).Encode(obj); err != nil { + log.Error("JSON encode: %v", err) + } +} + +func apiError(ctx *context.Context, status int, obj interface{}) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { + jsonResponse(ctx, status, map[string]string{ + "message": message, + }) + }) +} + +func baseURL(ctx *context.Context) string { + return setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/conan" +} + +// ExtractPathParameters is a middleware to extract common parameters from path +func ExtractPathParameters(ctx *context.Context) { + rref, err := conan_module.NewRecipeReference( + ctx.Params("name"), + ctx.Params("version"), + ctx.Params("user"), + ctx.Params("channel"), + ctx.Params("recipe_revision"), + ) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + + ctx.Data[recipeReferenceKey] = rref + + reference := ctx.Params("package_reference") + + var pref *conan_module.PackageReference + if reference != "" { + pref, err = conan_module.NewPackageReference( + rref, + reference, + ctx.Params("package_revision"), + ) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + } + + ctx.Data[packageReferenceKey] = pref +} + +// Ping reports the server capabilities +func Ping(ctx *context.Context) { + ctx.RespHeader().Add("X-Conan-Server-Capabilities", "revisions") // complex_search,checksum_deploy,matrix_params + + ctx.Status(http.StatusOK) +} + +// Authenticate creates an authentication token for the user +func Authenticate(ctx *context.Context) { + if ctx.Doer == nil { + apiError(ctx, http.StatusBadRequest, nil) + return + } + + token, err := packages_service.CreateAuthorizationToken(ctx.Doer) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.PlainText(http.StatusOK, token) +} + +// CheckCredentials tests if the provided authentication token is valid +func CheckCredentials(ctx *context.Context) { + if ctx.Doer == nil { + ctx.Status(http.StatusUnauthorized) + } else { + ctx.Status(http.StatusOK) + } +} + +// RecipeSnapshot displays the recipe files with their md5 hash +func RecipeSnapshot(ctx *context.Context) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + serveSnapshot(ctx, rref.AsKey()) +} + +// RecipeSnapshot displays the package files with their md5 hash +func PackageSnapshot(ctx *context.Context) { + pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) + + serveSnapshot(ctx, pref.AsKey()) +} + +func serveSnapshot(ctx *context.Context, fileKey string) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeConan, rref.Name, rref.Version) + if err != nil { + if err == packages_model.ErrPackageNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{ + VersionID: pv.ID, + CompositeKey: fileKey, + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pfs) == 0 { + apiError(ctx, http.StatusNotFound, nil) + return + } + + files := make(map[string]string) + for _, pf := range pfs { + pb, err := packages_model.GetBlobByID(ctx, pf.BlobID) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + files[pf.Name] = pb.HashMD5 + } + + jsonResponse(ctx, http.StatusOK, files) +} + +// RecipeDownloadURLs displays the recipe files with their download url +func RecipeDownloadURLs(ctx *context.Context) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + serveDownloadURLs( + ctx, + rref.AsKey(), + fmt.Sprintf(baseURL(ctx)+"/v1/files/%s/recipe", rref.LinkName()), + ) +} + +// PackageDownloadURLs displays the package files with their download url +func PackageDownloadURLs(ctx *context.Context) { + pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) + + serveDownloadURLs( + ctx, + pref.AsKey(), + fmt.Sprintf(baseURL(ctx)+"/v1/files/%s/package/%s", pref.Recipe.LinkName(), pref.LinkName()), + ) +} + +func serveDownloadURLs(ctx *context.Context, fileKey, downloadURL string) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeConan, rref.Name, rref.Version) + if err != nil { + if err == packages_model.ErrPackageNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{ + VersionID: pv.ID, + CompositeKey: fileKey, + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if len(pfs) == 0 { + apiError(ctx, http.StatusNotFound, nil) + return + } + + urls := make(map[string]string) + for _, pf := range pfs { + urls[pf.Name] = fmt.Sprintf("%s/%s", downloadURL, pf.Name) + } + + jsonResponse(ctx, http.StatusOK, urls) +} + +// RecipeUploadURLs displays the upload urls for the provided recipe files +func RecipeUploadURLs(ctx *context.Context) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + serveUploadURLs( + ctx, + recipeFileList, + fmt.Sprintf(baseURL(ctx)+"/v1/files/%s/recipe", rref.LinkName()), + ) +} + +// PackageUploadURLs displays the upload urls for the provided package files +func PackageUploadURLs(ctx *context.Context) { + pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) + + serveUploadURLs( + ctx, + packageFileList, + fmt.Sprintf(baseURL(ctx)+"/v1/files/%s/package/%s", pref.Recipe.LinkName(), pref.LinkName()), + ) +} + +func serveUploadURLs(ctx *context.Context, fileFilter stringSet, uploadURL string) { + defer ctx.Req.Body.Close() + + var files map[string]int64 + if err := json.NewDecoder(ctx.Req.Body).Decode(&files); err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + + urls := make(map[string]string) + for file := range files { + if _, ok := fileFilter[file]; ok { + urls[file] = fmt.Sprintf("%s/%s", uploadURL, file) + } + } + + jsonResponse(ctx, http.StatusOK, urls) +} + +// UploadRecipeFile handles the upload of a recipe file +func UploadRecipeFile(ctx *context.Context) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + uploadFile(ctx, recipeFileList, rref.AsKey()) +} + +// UploadPackageFile handles the upload of a package file +func UploadPackageFile(ctx *context.Context) { + pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) + + uploadFile(ctx, packageFileList, pref.AsKey()) +} + +func uploadFile(ctx *context.Context, fileFilter stringSet, fileKey string) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) + + filename := ctx.Params("filename") + if _, ok := fileFilter[filename]; !ok { + apiError(ctx, http.StatusBadRequest, nil) + return + } + + upload, close, err := ctx.UploadStream() + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + if close { + defer upload.Close() + } + + buf, err := packages_module.CreateHashedBufferFromReader(upload, 32*1024*1024) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + if buf.Size() == 0 { + // ignore empty uploads, second request contains content + jsonResponse(ctx, http.StatusOK, nil) + return + } + + isConanfileFile := filename == conanfileFile + + pci := &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeConan, + Name: rref.Name, + Version: rref.Version, + }, + SemverCompatible: true, + Creator: ctx.Doer, + } + pfci := &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: strings.ToLower(filename), + CompositeKey: fileKey, + }, + Data: buf, + IsLead: isConanfileFile, + Properties: map[string]string{ + conan_module.PropertyRecipeUser: rref.User, + conan_module.PropertyRecipeChannel: rref.Channel, + conan_module.PropertyRecipeRevision: rref.RevisionOrDefault(), + }, + OverwriteExisting: true, + } + + if pref != nil { + pfci.Properties[conan_module.PropertyPackageReference] = pref.Reference + pfci.Properties[conan_module.PropertyPackageRevision] = pref.RevisionOrDefault() + } + + if isConanfileFile || filename == conaninfoFile { + if isConanfileFile { + metadata, err := conan_module.ParseConanfile(buf) + if err != nil { + log.Error("Error parsing package metadata: %v", err) + apiError(ctx, http.StatusInternalServerError, err) + return + } + pv, err := packages_model.GetVersionByNameAndVersion(ctx, pci.Owner.ID, pci.PackageType, pci.Name, pci.Version) + if err != nil && err != packages_model.ErrPackageNotExist { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if pv != nil { + raw, err := json.Marshal(metadata) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + pv.MetadataJSON = string(raw) + if err := packages_model.UpdateVersion(ctx, pv); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + } else { + pci.Metadata = metadata + } + } else { + info, err := conan_module.ParseConaninfo(buf) + if err != nil { + log.Error("Error parsing conan info: %v", err) + apiError(ctx, http.StatusInternalServerError, err) + return + } + raw, err := json.Marshal(info) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + pfci.Properties[conan_module.PropertyPackageInfo] = string(raw) + } + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + } + + _, _, err = packages_service.CreatePackageOrAddFileToExisting( + pci, + pfci, + ) + if err != nil { + if err == packages_model.ErrDuplicatePackageFile { + apiError(ctx, http.StatusBadRequest, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.Status(http.StatusCreated) +} + +// DownloadRecipeFile serves the conent of the requested recipe file +func DownloadRecipeFile(ctx *context.Context) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + downloadFile(ctx, recipeFileList, rref.AsKey()) +} + +// DownloadPackageFile serves the conent of the requested package file +func DownloadPackageFile(ctx *context.Context) { + pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) + + downloadFile(ctx, packageFileList, pref.AsKey()) +} + +func downloadFile(ctx *context.Context, fileFilter stringSet, fileKey string) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + filename := ctx.Params("filename") + if _, ok := fileFilter[filename]; !ok { + apiError(ctx, http.StatusBadRequest, nil) + return + } + + s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + ctx, + &packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeConan, + Name: rref.Name, + Version: rref.Version, + }, + &packages_service.PackageFileInfo{ + Filename: filename, + CompositeKey: fileKey, + }, + ) + if err != nil { + if err == packages_model.ErrPackageNotExist || err == packages_model.ErrPackageFileNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer s.Close() + + ctx.ServeStream(s, pf.Name) +} + +// DeleteRecipeV1 deletes the requested recipe(s) +func DeleteRecipeV1(ctx *context.Context) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + if err := deleteRecipeOrPackage(ctx, rref, true, nil, false); err != nil { + if err == packages_model.ErrPackageNotExist || err == conan_model.ErrPackageReferenceNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + ctx.Status(http.StatusOK) +} + +// DeleteRecipeV2 deletes the requested recipe(s) respecting its revisions +func DeleteRecipeV2(ctx *context.Context) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + if err := deleteRecipeOrPackage(ctx, rref, rref.Revision == "", nil, false); err != nil { + if err == packages_model.ErrPackageNotExist || err == conan_model.ErrPackageReferenceNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + ctx.Status(http.StatusOK) +} + +// DeletePackageV1 deletes the requested package(s) +func DeletePackageV1(ctx *context.Context) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + type PackageReferences struct { + References []string `json:"package_ids"` + } + + var ids *PackageReferences + if err := json.NewDecoder(ctx.Req.Body).Decode(&ids); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + revisions, err := conan_model.GetRecipeRevisions(ctx, ctx.Package.Owner.ID, rref) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + for _, revision := range revisions { + currentRref := rref.WithRevision(revision.Value) + + var references []*conan_model.PropertyValue + if len(ids.References) == 0 { + if references, err = conan_model.GetPackageReferences(ctx, ctx.Package.Owner.ID, currentRref); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + } else { + for _, reference := range ids.References { + references = append(references, &conan_model.PropertyValue{Value: reference}) + } + } + + for _, reference := range references { + pref, _ := conan_module.NewPackageReference(currentRref, reference.Value, conan_module.DefaultRevision) + if err := deleteRecipeOrPackage(ctx, currentRref, true, pref, true); err != nil { + if err == packages_model.ErrPackageNotExist || err == conan_model.ErrPackageReferenceNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + } + } + ctx.Status(http.StatusOK) +} + +// DeletePackageV2 deletes the requested package(s) respecting its revisions +func DeletePackageV2(ctx *context.Context) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) + + if pref != nil { // has package reference + if err := deleteRecipeOrPackage(ctx, rref, false, pref, pref.Revision == ""); err != nil { + if err == packages_model.ErrPackageNotExist || err == conan_model.ErrPackageReferenceNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + } else { + ctx.Status(http.StatusOK) + } + return + } + + references, err := conan_model.GetPackageReferences(ctx, ctx.Package.Owner.ID, rref) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(references) == 0 { + apiError(ctx, http.StatusNotFound, conan_model.ErrPackageReferenceNotExist) + return + } + + for _, reference := range references { + pref, _ := conan_module.NewPackageReference(rref, reference.Value, conan_module.DefaultRevision) + + if err := deleteRecipeOrPackage(ctx, rref, false, pref, true); err != nil { + if err == packages_model.ErrPackageNotExist || err == conan_model.ErrPackageReferenceNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + } + + ctx.Status(http.StatusOK) +} + +func deleteRecipeOrPackage(apictx *context.Context, rref *conan_module.RecipeReference, ignoreRecipeRevision bool, pref *conan_module.PackageReference, ignorePackageRevision bool) error { + ctx, committer, err := db.TxContext() + if err != nil { + return err + } + defer committer.Close() + + pv, err := packages_model.GetVersionByNameAndVersion(ctx, apictx.Package.Owner.ID, packages_model.TypeConan, rref.Name, rref.Version) + if err != nil { + return err + } + + pd, err := packages_model.GetPackageDescriptor(ctx, pv) + if err != nil { + return err + } + + filter := map[string]string{ + conan_module.PropertyRecipeUser: rref.User, + conan_module.PropertyRecipeChannel: rref.Channel, + } + if !ignoreRecipeRevision { + filter[conan_module.PropertyRecipeRevision] = rref.RevisionOrDefault() + } + if pref != nil { + filter[conan_module.PropertyPackageReference] = pref.Reference + if !ignorePackageRevision { + filter[conan_module.PropertyPackageRevision] = pref.RevisionOrDefault() + } + } + + pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{ + VersionID: pv.ID, + Properties: filter, + }) + if err != nil { + return err + } + if len(pfs) == 0 { + return conan_model.ErrPackageReferenceNotExist + } + + for _, pf := range pfs { + if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil { + return err + } + if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil { + return err + } + } + + versionDeleted := false + has, err := packages_model.HasVersionFileReferences(ctx, pv.ID) + if err != nil { + return err + } + if !has { + versionDeleted = true + + if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeVersion, pv.ID); err != nil { + return err + } + + if err := packages_model.DeleteVersionByID(ctx, pv.ID); err != nil { + return err + } + } + + if err := committer.Commit(); err != nil { + return err + } + + if versionDeleted { + notification.NotifyPackageDelete(apictx.Doer, pd) + } + + return nil +} + +// ListRecipeRevisions gets a list of all recipe revisions +func ListRecipeRevisions(ctx *context.Context) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + revisions, err := conan_model.GetRecipeRevisions(ctx, ctx.Package.Owner.ID, rref) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + listRevisions(ctx, revisions) +} + +// ListPackageRevisions gets a list of all package revisions +func ListPackageRevisions(ctx *context.Context) { + pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) + + revisions, err := conan_model.GetPackageRevisions(ctx, ctx.Package.Owner.ID, pref) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + listRevisions(ctx, revisions) +} + +type revisionInfo struct { + Revision string `json:"revision"` + Time time.Time `json:"time"` +} + +func listRevisions(ctx *context.Context, revisions []*conan_model.PropertyValue) { + if len(revisions) == 0 { + apiError(ctx, http.StatusNotFound, conan_model.ErrRecipeReferenceNotExist) + return + } + + type RevisionList struct { + Revisions []*revisionInfo `json:"revisions"` + } + + revs := make([]*revisionInfo, 0, len(revisions)) + for _, rev := range revisions { + revs = append(revs, &revisionInfo{Revision: rev.Value, Time: time.Unix(int64(rev.CreatedUnix), 0)}) + } + + jsonResponse(ctx, http.StatusOK, &RevisionList{revs}) +} + +// LatestRecipeRevision gets the latest recipe revision +func LatestRecipeRevision(ctx *context.Context) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + revision, err := conan_model.GetLastRecipeRevision(ctx, ctx.Package.Owner.ID, rref) + if err != nil { + if err == conan_model.ErrRecipeReferenceNotExist || err == conan_model.ErrPackageReferenceNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + jsonResponse(ctx, http.StatusOK, &revisionInfo{Revision: revision.Value, Time: time.Unix(int64(revision.CreatedUnix), 0)}) +} + +// LatestPackageRevision gets the latest package revision +func LatestPackageRevision(ctx *context.Context) { + pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) + + revision, err := conan_model.GetLastPackageRevision(ctx, ctx.Package.Owner.ID, pref) + if err != nil { + if err == conan_model.ErrRecipeReferenceNotExist || err == conan_model.ErrPackageReferenceNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + jsonResponse(ctx, http.StatusOK, &revisionInfo{Revision: revision.Value, Time: time.Unix(int64(revision.CreatedUnix), 0)}) +} + +// ListRecipeRevisionFiles gets a list of all recipe revision files +func ListRecipeRevisionFiles(ctx *context.Context) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + listRevisionFiles(ctx, rref.AsKey()) +} + +// ListPackageRevisionFiles gets a list of all package revision files +func ListPackageRevisionFiles(ctx *context.Context) { + pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) + + listRevisionFiles(ctx, pref.AsKey()) +} + +func listRevisionFiles(ctx *context.Context, fileKey string) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeConan, rref.Name, rref.Version) + if err != nil { + if err == packages_model.ErrPackageNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{ + VersionID: pv.ID, + CompositeKey: fileKey, + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pfs) == 0 { + apiError(ctx, http.StatusNotFound, nil) + return + } + + files := make(map[string]interface{}) + for _, pf := range pfs { + files[pf.Name] = nil + } + + type FileList struct { + Files map[string]interface{} `json:"files"` + } + + jsonResponse(ctx, http.StatusOK, &FileList{ + Files: files, + }) +} diff --git a/routers/api/packages/conan/search.go b/routers/api/packages/conan/search.go new file mode 100644 index 0000000000..39dd6362aa --- /dev/null +++ b/routers/api/packages/conan/search.go @@ -0,0 +1,164 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package conan + +import ( + "net/http" + "strings" + + conan_model "code.gitea.io/gitea/models/packages/conan" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/json" + conan_module "code.gitea.io/gitea/modules/packages/conan" +) + +// SearchResult contains the found recipe names +type SearchResult struct { + Results []string `json:"results"` +} + +// SearchRecipes searches all recipes matching the query +func SearchRecipes(ctx *context.Context) { + q := ctx.FormTrim("q") + + opts := parseQuery(ctx.Package.Owner, q) + + results, err := conan_model.SearchRecipes(ctx, opts) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + jsonResponse(ctx, http.StatusOK, &SearchResult{ + Results: results, + }) +} + +// parseQuery creates search options for the given query +func parseQuery(owner *user_model.User, query string) *conan_model.RecipeSearchOptions { + opts := &conan_model.RecipeSearchOptions{ + OwnerID: owner.ID, + } + + if query != "" { + parts := strings.Split(strings.ReplaceAll(query, "@", "/"), "/") + + opts.Name = parts[0] + if len(parts) > 1 && parts[1] != "*" { + opts.Version = parts[1] + } + if len(parts) > 2 && parts[2] != "*" { + opts.User = parts[2] + } + if len(parts) > 3 && parts[3] != "*" { + opts.Channel = parts[3] + } + } + + return opts +} + +// SearchPackagesV1 searches all packages of a recipe (Conan v1 endpoint) +func SearchPackagesV1(ctx *context.Context) { + searchPackages(ctx, true) +} + +// SearchPackagesV2 searches all packages of a recipe (Conan v2 endpoint) +func SearchPackagesV2(ctx *context.Context) { + searchPackages(ctx, false) +} + +func searchPackages(ctx *context.Context, searchAllRevisions bool) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + if !searchAllRevisions && rref.Revision == "" { + lastRevision, err := conan_model.GetLastRecipeRevision(ctx, ctx.Package.Owner.ID, rref) + if err != nil { + if err == conan_model.ErrRecipeReferenceNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + rref = rref.WithRevision(lastRevision.Value) + } else { + has, err := conan_model.RecipeExists(ctx, ctx.Package.Owner.ID, rref) + if err != nil { + if err == conan_model.ErrRecipeReferenceNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + if !has { + apiError(ctx, http.StatusNotFound, nil) + return + } + } + + recipeRevisions := []*conan_model.PropertyValue{{Value: rref.Revision}} + if searchAllRevisions { + var err error + recipeRevisions, err = conan_model.GetRecipeRevisions(ctx, ctx.Package.Owner.ID, rref) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + } + + result := make(map[string]*conan_module.Conaninfo) + + for _, recipeRevision := range recipeRevisions { + currentRef := rref + if recipeRevision.Value != "" { + currentRef = rref.WithRevision(recipeRevision.Value) + } + packageReferences, err := conan_model.GetPackageReferences(ctx, ctx.Package.Owner.ID, currentRef) + if err != nil { + if err == conan_model.ErrRecipeReferenceNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + for _, packageReference := range packageReferences { + if _, ok := result[packageReference.Value]; ok { + continue + } + pref, _ := conan_module.NewPackageReference(currentRef, packageReference.Value, "") + lastPackageRevision, err := conan_model.GetLastPackageRevision(ctx, ctx.Package.Owner.ID, pref) + if err != nil { + if err == conan_model.ErrPackageReferenceNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + pref = pref.WithRevision(lastPackageRevision.Value) + infoRaw, err := conan_model.GetPackageInfo(ctx, ctx.Package.Owner.ID, pref) + if err != nil { + if err == conan_model.ErrPackageReferenceNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + var info *conan_module.Conaninfo + if err := json.Unmarshal([]byte(infoRaw), &info); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + result[pref.Reference] = info + } + } + + jsonResponse(ctx, http.StatusOK, result) +} diff --git a/routers/api/packages/container/auth.go b/routers/api/packages/container/auth.go new file mode 100644 index 0000000000..770068a3bf --- /dev/null +++ b/routers/api/packages/container/auth.go @@ -0,0 +1,45 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package container + +import ( + "net/http" + + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/services/auth" + "code.gitea.io/gitea/services/packages" +) + +type Auth struct{} + +func (a *Auth) Name() string { + return "container" +} + +// Verify extracts the user from the Bearer token +// If it's an anonymous session a ghost user is returned +func (a *Auth) Verify(req *http.Request, w http.ResponseWriter, store auth.DataStore, sess auth.SessionStore) *user_model.User { + uid, err := packages.ParseAuthorizationToken(req) + if err != nil { + log.Trace("ParseAuthorizationToken: %v", err) + return nil + } + + if uid == 0 { + return nil + } + if uid == -1 { + return user_model.NewGhostUser() + } + + u, err := user_model.GetUserByID(uid) + if err != nil { + log.Error("GetUserByID: %v", err) + return nil + } + + return u +} diff --git a/routers/api/packages/container/blob.go b/routers/api/packages/container/blob.go new file mode 100644 index 0000000000..8f6254f583 --- /dev/null +++ b/routers/api/packages/container/blob.go @@ -0,0 +1,136 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package container + +import ( + "context" + "encoding/hex" + "fmt" + "strings" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + container_model "code.gitea.io/gitea/models/packages/container" + "code.gitea.io/gitea/modules/log" + packages_module "code.gitea.io/gitea/modules/packages" + container_module "code.gitea.io/gitea/modules/packages/container" + packages_service "code.gitea.io/gitea/services/packages" +) + +// saveAsPackageBlob creates a package blob from an upload +// The uploaded blob gets stored in a special upload version to link them to the package/image +func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_service.PackageInfo) (*packages_model.PackageBlob, error) { + pb := packages_service.NewPackageBlob(hsr) + + exists := false + + contentStore := packages_module.NewContentStore() + + err := db.WithTx(func(ctx context.Context) error { + p := &packages_model.Package{ + OwnerID: pi.Owner.ID, + Type: packages_model.TypeContainer, + Name: strings.ToLower(pi.Name), + LowerName: strings.ToLower(pi.Name), + } + var err error + if p, err = packages_model.TryInsertPackage(ctx, p); err != nil { + if err != packages_model.ErrDuplicatePackage { + log.Error("Error inserting package: %v", err) + return err + } + } + + pv := &packages_model.PackageVersion{ + PackageID: p.ID, + CreatorID: pi.Owner.ID, + Version: container_model.UploadVersion, + LowerVersion: container_model.UploadVersion, + IsInternal: true, + MetadataJSON: "null", + } + if pv, err = packages_model.GetOrInsertVersion(ctx, pv); err != nil { + if err != packages_model.ErrDuplicatePackageVersion { + log.Error("Error inserting package: %v", err) + return err + } + } + + pb, exists, err = packages_model.GetOrInsertBlob(ctx, pb) + if err != nil { + log.Error("Error inserting package blob: %v", err) + return err + } + if !exists { + if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), hsr, hsr.Size()); err != nil { + log.Error("Error saving package blob in content store: %v", err) + return err + } + } + + filename := strings.ToLower(fmt.Sprintf("sha256_%s", pb.HashSHA256)) + + pf := &packages_model.PackageFile{ + VersionID: pv.ID, + BlobID: pb.ID, + Name: filename, + LowerName: filename, + CompositeKey: packages_model.EmptyFileKey, + } + if pf, err = packages_model.TryInsertFile(ctx, pf); err != nil { + if err == packages_model.ErrDuplicatePackageFile { + return nil + } + log.Error("Error inserting package file: %v", err) + return err + } + + if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeFile, pf.ID, container_module.PropertyDigest, digestFromPackageBlob(pb)); err != nil { + log.Error("Error setting package file property: %v", err) + return err + } + + return nil + }) + if err != nil { + if !exists { + if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil { + log.Error("Error deleting package blob from content store: %v", err) + } + } + return nil, err + } + + return pb, nil +} + +func deleteBlob(ownerID int64, image, digest string) error { + return db.WithTx(func(ctx context.Context) error { + pfds, err := container_model.GetContainerBlobs(ctx, &container_model.BlobSearchOptions{ + OwnerID: ownerID, + Image: image, + Digest: digest, + }) + if err != nil { + return err + } + + for _, file := range pfds { + if err := packages_service.DeletePackageFile(ctx, file.File); err != nil { + return err + } + } + return nil + }) +} + +func digestFromHashSummer(h packages_module.HashSummer) string { + _, _, hashSHA256, _ := h.Sums() + return "sha256:" + hex.EncodeToString(hashSHA256) +} + +func digestFromPackageBlob(pb *packages_model.PackageBlob) string { + return "sha256:" + pb.HashSHA256 +} diff --git a/routers/api/packages/container/container.go b/routers/api/packages/container/container.go new file mode 100644 index 0000000000..f0b1fafd26 --- /dev/null +++ b/routers/api/packages/container/container.go @@ -0,0 +1,613 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package container + +import ( + "errors" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + + packages_model "code.gitea.io/gitea/models/packages" + container_model "code.gitea.io/gitea/models/packages/container" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + packages_module "code.gitea.io/gitea/modules/packages" + container_module "code.gitea.io/gitea/modules/packages/container" + "code.gitea.io/gitea/modules/packages/container/oci" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/routers/api/packages/helper" + packages_service "code.gitea.io/gitea/services/packages" + container_service "code.gitea.io/gitea/services/packages/container" +) + +// maximum size of a container manifest +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-manifests +const maxManifestSize = 10 * 1024 * 1024 + +var imageNamePattern = regexp.MustCompile(`\A[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*\z`) + +type containerHeaders struct { + Status int + ContentDigest string + UploadUUID string + Range string + Location string + ContentType string + ContentLength int64 +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#legacy-docker-support-http-headers +func setResponseHeaders(resp http.ResponseWriter, h *containerHeaders) { + if h.Location != "" { + resp.Header().Set("Location", h.Location) + } + if h.Range != "" { + resp.Header().Set("Range", h.Range) + } + if h.ContentType != "" { + resp.Header().Set("Content-Type", h.ContentType) + } + if h.ContentLength != 0 { + resp.Header().Set("Content-Length", strconv.FormatInt(h.ContentLength, 10)) + } + if h.UploadUUID != "" { + resp.Header().Set("Docker-Upload-Uuid", h.UploadUUID) + } + if h.ContentDigest != "" { + resp.Header().Set("Docker-Content-Digest", h.ContentDigest) + resp.Header().Set("ETag", fmt.Sprintf(`"%s"`, h.ContentDigest)) + } + resp.Header().Set("Docker-Distribution-Api-Version", "registry/2.0") + resp.WriteHeader(h.Status) +} + +func jsonResponse(ctx *context.Context, status int, obj interface{}) { + setResponseHeaders(ctx.Resp, &containerHeaders{ + Status: status, + ContentType: "application/json", + }) + if err := json.NewEncoder(ctx.Resp).Encode(obj); err != nil { + log.Error("JSON encode: %v", err) + } +} + +func apiError(ctx *context.Context, status int, err error) { + helper.LogAndProcessError(ctx, status, err, func(message string) { + setResponseHeaders(ctx.Resp, &containerHeaders{ + Status: status, + }) + }) +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#error-codes +func apiErrorDefined(ctx *context.Context, err *namedError) { + type ContainerError struct { + Code string `json:"code"` + Message string `json:"message"` + } + + type ContainerErrors struct { + Errors []ContainerError `json:"errors"` + } + + jsonResponse(ctx, err.StatusCode, ContainerErrors{ + Errors: []ContainerError{ + { + Code: err.Code, + Message: err.Message, + }, + }, + }) +} + +// ReqContainerAccess is a middleware which checks the current user valid (real user or ghost for anonymous access) +func ReqContainerAccess(ctx *context.Context) { + if ctx.Doer == nil { + ctx.Resp.Header().Add("WWW-Authenticate", `Bearer realm="`+setting.AppURL+`v2/token"`) + ctx.Resp.Header().Add("WWW-Authenticate", `Basic`) + apiErrorDefined(ctx, errUnauthorized) + } +} + +// VerifyImageName is a middleware which checks if the image name is allowed +func VerifyImageName(ctx *context.Context) { + if !imageNamePattern.MatchString(ctx.Params("image")) { + apiErrorDefined(ctx, errNameInvalid) + } +} + +// DetermineSupport is used to test if the registry supports OCI +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#determining-support +func DetermineSupport(ctx *context.Context) { + setResponseHeaders(ctx.Resp, &containerHeaders{ + Status: http.StatusOK, + }) +} + +// Authenticate creates a token for the current user +// If the current user is anonymous, the ghost user is used +func Authenticate(ctx *context.Context) { + u := ctx.Doer + if u == nil { + u = user_model.NewGhostUser() + } + + token, err := packages_service.CreateAuthorizationToken(u) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.JSON(http.StatusOK, map[string]string{ + "token": token, + }) +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#mounting-a-blob-from-another-repository +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#single-post +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-a-blob-in-chunks +func InitiateUploadBlob(ctx *context.Context) { + image := ctx.Params("image") + + mount := ctx.FormTrim("mount") + from := ctx.FormTrim("from") + if mount != "" { + blob, _ := container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{ + Image: from, + Digest: mount, + }) + if blob != nil { + setResponseHeaders(ctx.Resp, &containerHeaders{ + Location: fmt.Sprintf("/v2/%s/%s/blobs/%s", ctx.Package.Owner.LowerName, image, mount), + ContentDigest: mount, + Status: http.StatusCreated, + }) + return + } + } + + digest := ctx.FormTrim("digest") + if digest != "" { + buf, err := packages_module.CreateHashedBufferFromReader(ctx.Req.Body, 32*1024*1024) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + if digest != digestFromHashSummer(buf) { + apiErrorDefined(ctx, errDigestInvalid) + return + } + + if _, err := saveAsPackageBlob(buf, &packages_service.PackageInfo{Owner: ctx.Package.Owner, Name: image}); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + setResponseHeaders(ctx.Resp, &containerHeaders{ + Location: fmt.Sprintf("/v2/%s/%s/blobs/%s", ctx.Package.Owner.LowerName, image, digest), + ContentDigest: digest, + Status: http.StatusCreated, + }) + return + } + + upload, err := packages_model.CreateBlobUpload(ctx) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + setResponseHeaders(ctx.Resp, &containerHeaders{ + Location: fmt.Sprintf("/v2/%s/%s/blobs/uploads/%s", ctx.Package.Owner.LowerName, image, upload.ID), + Range: "0-0", + UploadUUID: upload.ID, + Status: http.StatusAccepted, + }) +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-a-blob-in-chunks +func UploadBlob(ctx *context.Context) { + image := ctx.Params("image") + + uploader, err := container_service.NewBlobUploader(ctx, ctx.Params("uuid")) + if err != nil { + if err == packages_model.ErrPackageBlobUploadNotExist { + apiErrorDefined(ctx, errBlobUploadUnknown) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + defer uploader.Close() + + contentRange := ctx.Req.Header.Get("Content-Range") + if contentRange != "" { + start, end := 0, 0 + if _, err := fmt.Sscanf(contentRange, "%d-%d", &start, &end); err != nil { + apiErrorDefined(ctx, errBlobUploadInvalid) + return + } + + if int64(start) != uploader.Size() { + apiErrorDefined(ctx, errBlobUploadInvalid.WithStatusCode(http.StatusRequestedRangeNotSatisfiable)) + return + } + } else if uploader.Size() != 0 { + apiErrorDefined(ctx, errBlobUploadInvalid.WithMessage("Stream uploads after first write are not allowed")) + return + } + + if err := uploader.Append(ctx, ctx.Req.Body); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + setResponseHeaders(ctx.Resp, &containerHeaders{ + Location: fmt.Sprintf("/v2/%s/%s/blobs/uploads/%s", ctx.Package.Owner.LowerName, image, uploader.ID), + Range: fmt.Sprintf("0-%d", uploader.Size()-1), + UploadUUID: uploader.ID, + Status: http.StatusAccepted, + }) +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-a-blob-in-chunks +func EndUploadBlob(ctx *context.Context) { + image := ctx.Params("image") + + digest := ctx.FormTrim("digest") + if digest == "" { + apiErrorDefined(ctx, errDigestInvalid) + return + } + + uploader, err := container_service.NewBlobUploader(ctx, ctx.Params("uuid")) + if err != nil { + if err == packages_model.ErrPackageBlobUploadNotExist { + apiErrorDefined(ctx, errBlobUploadUnknown) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + close := true + defer func() { + if close { + uploader.Close() + } + }() + + if ctx.Req.Body != nil { + if err := uploader.Append(ctx, ctx.Req.Body); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + } + + if digest != digestFromHashSummer(uploader) { + apiErrorDefined(ctx, errDigestInvalid) + return + } + + if _, err := saveAsPackageBlob(uploader, &packages_service.PackageInfo{Owner: ctx.Package.Owner, Name: image}); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if err := uploader.Close(); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + close = false + + if err := container_service.RemoveBlobUploadByID(ctx, uploader.ID); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + setResponseHeaders(ctx.Resp, &containerHeaders{ + Location: fmt.Sprintf("/v2/%s/%s/blobs/%s", ctx.Package.Owner.LowerName, image, digest), + ContentDigest: digest, + Status: http.StatusCreated, + }) +} + +func getBlobFromContext(ctx *context.Context) (*packages_model.PackageFileDescriptor, error) { + digest := ctx.Params("digest") + + if !oci.Digest(digest).Validate() { + return nil, container_model.ErrContainerBlobNotExist + } + + return container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + Image: ctx.Params("image"), + Digest: digest, + }) +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#checking-if-content-exists-in-the-registry +func HeadBlob(ctx *context.Context) { + blob, err := getBlobFromContext(ctx) + if err != nil { + if err == container_model.ErrContainerBlobNotExist { + apiErrorDefined(ctx, errBlobUnknown) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + setResponseHeaders(ctx.Resp, &containerHeaders{ + ContentDigest: blob.Properties.GetByName(container_module.PropertyDigest), + ContentLength: blob.Blob.Size, + Status: http.StatusOK, + }) +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pulling-blobs +func GetBlob(ctx *context.Context) { + blob, err := getBlobFromContext(ctx) + if err != nil { + if err == container_model.ErrContainerBlobNotExist { + apiErrorDefined(ctx, errBlobUnknown) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + s, err := packages_module.NewContentStore().Get(packages_module.BlobHash256Key(blob.Blob.HashSHA256)) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer s.Close() + + setResponseHeaders(ctx.Resp, &containerHeaders{ + ContentDigest: blob.Properties.GetByName(container_module.PropertyDigest), + ContentType: blob.Properties.GetByName(container_module.PropertyMediaType), + ContentLength: blob.Blob.Size, + Status: http.StatusOK, + }) + if _, err := io.Copy(ctx.Resp, s); err != nil { + log.Error("Error whilst copying content to response: %v", err) + } +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#deleting-blobs +func DeleteBlob(ctx *context.Context) { + digest := ctx.Params("digest") + + if !oci.Digest(digest).Validate() { + apiErrorDefined(ctx, errBlobUnknown) + return + } + + if err := deleteBlob(ctx.Package.Owner.ID, ctx.Params("image"), digest); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + setResponseHeaders(ctx.Resp, &containerHeaders{ + Status: http.StatusAccepted, + }) +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-manifests +func UploadManifest(ctx *context.Context) { + reference := ctx.Params("reference") + + mci := &manifestCreationInfo{ + MediaType: oci.MediaType(ctx.Req.Header.Get("Content-Type")), + Owner: ctx.Package.Owner, + Creator: ctx.Doer, + Image: ctx.Params("image"), + Reference: reference, + IsTagged: !oci.Digest(reference).Validate(), + } + + if mci.IsTagged && !oci.Reference(reference).Validate() { + apiErrorDefined(ctx, errManifestInvalid.WithMessage("Tag is invalid")) + return + } + + maxSize := maxManifestSize + 1 + buf, err := packages_module.CreateHashedBufferFromReader(&io.LimitedReader{R: ctx.Req.Body, N: int64(maxSize)}, maxSize) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + if buf.Size() > maxManifestSize { + apiErrorDefined(ctx, errManifestInvalid.WithMessage("Manifest exceeds maximum size").WithStatusCode(http.StatusRequestEntityTooLarge)) + return + } + + digest, err := processManifest(mci, buf) + if err != nil { + var namedError *namedError + if errors.As(err, &namedError) { + apiErrorDefined(ctx, namedError) + } else if errors.Is(err, container_model.ErrContainerBlobNotExist) { + apiErrorDefined(ctx, errBlobUnknown) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + setResponseHeaders(ctx.Resp, &containerHeaders{ + Location: fmt.Sprintf("/v2/%s/%s/manifests/%s", ctx.Package.Owner.LowerName, mci.Image, reference), + ContentDigest: digest, + Status: http.StatusCreated, + }) +} + +func getManifestFromContext(ctx *context.Context) (*packages_model.PackageFileDescriptor, error) { + reference := ctx.Params("reference") + + opts := &container_model.BlobSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + Image: ctx.Params("image"), + IsManifest: true, + } + if oci.Digest(reference).Validate() { + opts.Digest = reference + } else if oci.Reference(reference).Validate() { + opts.Tag = reference + } else { + return nil, container_model.ErrContainerBlobNotExist + } + + return container_model.GetContainerBlob(ctx, opts) +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#checking-if-content-exists-in-the-registry +func HeadManifest(ctx *context.Context) { + manifest, err := getManifestFromContext(ctx) + if err != nil { + if err == container_model.ErrContainerBlobNotExist { + apiErrorDefined(ctx, errManifestUnknown) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + setResponseHeaders(ctx.Resp, &containerHeaders{ + ContentDigest: manifest.Properties.GetByName(container_module.PropertyDigest), + ContentType: manifest.Properties.GetByName(container_module.PropertyMediaType), + ContentLength: manifest.Blob.Size, + Status: http.StatusOK, + }) +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pulling-manifests +func GetManifest(ctx *context.Context) { + manifest, err := getManifestFromContext(ctx) + if err != nil { + if err == container_model.ErrContainerBlobNotExist { + apiErrorDefined(ctx, errManifestUnknown) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + s, err := packages_module.NewContentStore().Get(packages_module.BlobHash256Key(manifest.Blob.HashSHA256)) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer s.Close() + + setResponseHeaders(ctx.Resp, &containerHeaders{ + ContentDigest: manifest.Properties.GetByName(container_module.PropertyDigest), + ContentType: manifest.Properties.GetByName(container_module.PropertyMediaType), + ContentLength: manifest.Blob.Size, + Status: http.StatusOK, + }) + if _, err := io.Copy(ctx.Resp, s); err != nil { + log.Error("Error whilst copying content to response: %v", err) + } +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#deleting-tags +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#deleting-manifests +func DeleteManifest(ctx *context.Context) { + reference := ctx.Params("reference") + + opts := &container_model.BlobSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + Image: ctx.Params("image"), + IsManifest: true, + } + if oci.Digest(reference).Validate() { + opts.Digest = reference + } else if oci.Reference(reference).Validate() { + opts.Tag = reference + } else { + apiErrorDefined(ctx, errManifestUnknown) + return + } + + pvs, err := container_model.GetManifestVersions(ctx, opts) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if len(pvs) == 0 { + apiErrorDefined(ctx, errManifestUnknown) + return + } + + for _, pv := range pvs { + if err := packages_service.RemovePackageVersion(ctx.Doer, pv); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + } + + setResponseHeaders(ctx.Resp, &containerHeaders{ + Status: http.StatusAccepted, + }) +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#content-discovery +func GetTagList(ctx *context.Context) { + image := ctx.Params("image") + + if _, err := packages_model.GetPackageByName(ctx, ctx.Package.Owner.ID, packages_model.TypeContainer, image); err != nil { + if err == packages_model.ErrPackageNotExist { + apiErrorDefined(ctx, errNameUnknown) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + n := -1 + if ctx.FormTrim("n") != "" { + n = ctx.FormInt("n") + } + last := ctx.FormTrim("last") + + tags, err := container_model.GetImageTags(ctx, ctx.Package.Owner.ID, image, n, last) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + type TagList struct { + Name string `json:"name"` + Tags []string `json:"tags"` + } + + if len(tags) > 0 { + v := url.Values{} + if n > 0 { + v.Add("n", strconv.Itoa(n)) + } + v.Add("last", tags[len(tags)-1]) + + ctx.Resp.Header().Set("Link", fmt.Sprintf(`</v2/%s/%s/tags/list?%s>; rel="next"`, ctx.Package.Owner.LowerName, image, v.Encode())) + } + + jsonResponse(ctx, http.StatusOK, TagList{ + Name: strings.ToLower(ctx.Package.Owner.LowerName + "/" + image), + Tags: tags, + }) +} diff --git a/routers/api/packages/container/errors.go b/routers/api/packages/container/errors.go new file mode 100644 index 0000000000..0efbb081ca --- /dev/null +++ b/routers/api/packages/container/errors.go @@ -0,0 +1,53 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package container + +import ( + "net/http" +) + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#error-codes +var ( + errBlobUnknown = &namedError{Code: "BLOB_UNKNOWN", StatusCode: http.StatusNotFound} + errBlobUploadInvalid = &namedError{Code: "BLOB_UPLOAD_INVALID", StatusCode: http.StatusBadRequest} + errBlobUploadUnknown = &namedError{Code: "BLOB_UPLOAD_UNKNOWN", StatusCode: http.StatusNotFound} + errDigestInvalid = &namedError{Code: "DIGEST_INVALID", StatusCode: http.StatusBadRequest} + errManifestBlobUnknown = &namedError{Code: "MANIFEST_BLOB_UNKNOWN", StatusCode: http.StatusNotFound} + errManifestInvalid = &namedError{Code: "MANIFEST_INVALID", StatusCode: http.StatusBadRequest} + errManifestUnknown = &namedError{Code: "MANIFEST_UNKNOWN", StatusCode: http.StatusNotFound} + errNameInvalid = &namedError{Code: "NAME_INVALID", StatusCode: http.StatusBadRequest} + errNameUnknown = &namedError{Code: "NAME_UNKNOWN", StatusCode: http.StatusNotFound} + errSizeInvalid = &namedError{Code: "SIZE_INVALID", StatusCode: http.StatusBadRequest} + errUnauthorized = &namedError{Code: "UNAUTHORIZED", StatusCode: http.StatusUnauthorized} + errUnsupported = &namedError{Code: "UNSUPPORTED", StatusCode: http.StatusNotImplemented} +) + +type namedError struct { + Code string + StatusCode int + Message string +} + +func (e *namedError) Error() string { + return e.Message +} + +// WithMessage creates a new instance of the error with a different message +func (e *namedError) WithMessage(message string) *namedError { + return &namedError{ + Code: e.Code, + StatusCode: e.StatusCode, + Message: message, + } +} + +// WithStatusCode creates a new instance of the error with a different status code +func (e *namedError) WithStatusCode(statusCode int) *namedError { + return &namedError{ + Code: e.Code, + StatusCode: statusCode, + Message: e.Message, + } +} diff --git a/routers/api/packages/container/manifest.go b/routers/api/packages/container/manifest.go new file mode 100644 index 0000000000..b327538e6f --- /dev/null +++ b/routers/api/packages/container/manifest.go @@ -0,0 +1,408 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package container + +import ( + "context" + "fmt" + "io" + "strings" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + container_model "code.gitea.io/gitea/models/packages/container" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + packages_module "code.gitea.io/gitea/modules/packages" + container_module "code.gitea.io/gitea/modules/packages/container" + "code.gitea.io/gitea/modules/packages/container/oci" + packages_service "code.gitea.io/gitea/services/packages" +) + +// manifestCreationInfo describes a manifest to create +type manifestCreationInfo struct { + MediaType oci.MediaType + Owner *user_model.User + Creator *user_model.User + Image string + Reference string + IsTagged bool + Properties map[string]string +} + +func processManifest(mci *manifestCreationInfo, buf *packages_module.HashedBuffer) (string, error) { + var schema oci.SchemaMediaBase + if err := json.NewDecoder(buf).Decode(&schema); err != nil { + return "", err + } + + if schema.SchemaVersion != 2 { + return "", errUnsupported.WithMessage("Schema version is not supported") + } + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + return "", err + } + + if !mci.MediaType.IsValid() { + mci.MediaType = schema.MediaType + if !mci.MediaType.IsValid() { + return "", errManifestInvalid.WithMessage("MediaType not recognized") + } + } + + if mci.MediaType.IsImageManifest() { + d, err := processImageManifest(mci, buf) + return d, err + } else if mci.MediaType.IsImageIndex() { + d, err := processImageManifestIndex(mci, buf) + return d, err + } + return "", errManifestInvalid +} + +func processImageManifest(mci *manifestCreationInfo, buf *packages_module.HashedBuffer) (string, error) { + manifestDigest := "" + + err := func() error { + var manifest oci.Manifest + if err := json.NewDecoder(buf).Decode(&manifest); err != nil { + return err + } + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + return err + } + + ctx, committer, err := db.TxContext() + if err != nil { + return err + } + defer committer.Close() + + configDescriptor, err := container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{ + OwnerID: mci.Owner.ID, + Image: mci.Image, + Digest: string(manifest.Config.Digest), + }) + if err != nil { + return err + } + + configReader, err := packages_module.NewContentStore().Get(packages_module.BlobHash256Key(configDescriptor.Blob.HashSHA256)) + if err != nil { + return err + } + defer configReader.Close() + + metadata, err := container_module.ParseImageConfig(manifest.Config.MediaType, configReader) + if err != nil { + return err + } + + blobReferences := make([]*blobReference, 0, 1+len(manifest.Layers)) + + blobReferences = append(blobReferences, &blobReference{ + Digest: manifest.Config.Digest, + MediaType: manifest.Config.MediaType, + File: configDescriptor, + ExpectedSize: manifest.Config.Size, + }) + + for _, layer := range manifest.Layers { + pfd, err := container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{ + OwnerID: mci.Owner.ID, + Image: mci.Image, + Digest: string(layer.Digest), + }) + if err != nil { + return err + } + + blobReferences = append(blobReferences, &blobReference{ + Digest: layer.Digest, + MediaType: layer.MediaType, + File: pfd, + ExpectedSize: layer.Size, + }) + } + + pv, err := createPackageAndVersion(ctx, mci, metadata) + if err != nil { + return err + } + + uploadVersion, err := packages_model.GetInternalVersionByNameAndVersion(ctx, mci.Owner.ID, packages_model.TypeContainer, mci.Image, container_model.UploadVersion) + if err != nil && err != packages_model.ErrPackageNotExist { + return err + } + + for _, ref := range blobReferences { + if err := createFileFromBlobReference(ctx, pv, uploadVersion, ref); err != nil { + return err + } + } + + pb, created, digest, err := createManifestBlob(ctx, mci, pv, buf) + removeBlob := false + defer func() { + if removeBlob { + contentStore := packages_module.NewContentStore() + if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil { + log.Error("Error deleting package blob from content store: %v", err) + } + } + }() + if err != nil { + removeBlob = created + return err + } + + if err := committer.Commit(); err != nil { + removeBlob = created + return err + } + + manifestDigest = digest + + return nil + }() + if err != nil { + return "", err + } + + return manifestDigest, nil +} + +func processImageManifestIndex(mci *manifestCreationInfo, buf *packages_module.HashedBuffer) (string, error) { + manifestDigest := "" + + err := func() error { + var index oci.Index + if err := json.NewDecoder(buf).Decode(&index); err != nil { + return err + } + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + return err + } + + ctx, committer, err := db.TxContext() + if err != nil { + return err + } + defer committer.Close() + + metadata := &container_module.Metadata{ + Type: container_module.TypeOCI, + MultiArch: make(map[string]string), + } + + for _, manifest := range index.Manifests { + if !manifest.MediaType.IsImageManifest() { + return errManifestInvalid + } + + platform := container_module.DefaultPlatform + if manifest.Platform != nil { + platform = fmt.Sprintf("%s/%s", manifest.Platform.OS, manifest.Platform.Architecture) + if manifest.Platform.Variant != "" { + platform = fmt.Sprintf("%s/%s", platform, manifest.Platform.Variant) + } + } + + _, err := container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{ + OwnerID: mci.Owner.ID, + Image: mci.Image, + Digest: string(manifest.Digest), + IsManifest: true, + }) + if err != nil { + if err == container_model.ErrContainerBlobNotExist { + return errManifestBlobUnknown + } + return err + } + + metadata.MultiArch[platform] = string(manifest.Digest) + } + + pv, err := createPackageAndVersion(ctx, mci, metadata) + if err != nil { + return err + } + + pb, created, digest, err := createManifestBlob(ctx, mci, pv, buf) + removeBlob := false + defer func() { + if removeBlob { + contentStore := packages_module.NewContentStore() + if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil { + log.Error("Error deleting package blob from content store: %v", err) + } + } + }() + if err != nil { + removeBlob = created + return err + } + + if err := committer.Commit(); err != nil { + removeBlob = created + return err + } + + manifestDigest = digest + + return nil + }() + if err != nil { + return "", err + } + + return manifestDigest, nil +} + +func createPackageAndVersion(ctx context.Context, mci *manifestCreationInfo, metadata *container_module.Metadata) (*packages_model.PackageVersion, error) { + p := &packages_model.Package{ + OwnerID: mci.Owner.ID, + Type: packages_model.TypeContainer, + Name: strings.ToLower(mci.Image), + LowerName: strings.ToLower(mci.Image), + } + var err error + if p, err = packages_model.TryInsertPackage(ctx, p); err != nil { + if err != packages_model.ErrDuplicatePackage { + log.Error("Error inserting package: %v", err) + return nil, err + } + } + + metadata.IsTagged = mci.IsTagged + + metadataJSON, err := json.Marshal(metadata) + if err != nil { + return nil, err + } + + _pv := &packages_model.PackageVersion{ + PackageID: p.ID, + CreatorID: mci.Creator.ID, + Version: strings.ToLower(mci.Reference), + LowerVersion: strings.ToLower(mci.Reference), + MetadataJSON: string(metadataJSON), + } + var pv *packages_model.PackageVersion + if pv, err = packages_model.GetOrInsertVersion(ctx, _pv); err != nil { + if err == packages_model.ErrDuplicatePackageVersion { + if err := packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil { + return nil, err + } + + if pv, err = packages_model.GetOrInsertVersion(ctx, _pv); err != nil { + log.Error("Error inserting package: %v", err) + return nil, err + } + } else { + log.Error("Error inserting package: %v", err) + return nil, err + } + } + + if mci.IsTagged { + if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestTagged, ""); err != nil { + log.Error("Error setting package version property: %v", err) + return nil, err + } + } + for _, digest := range metadata.MultiArch { + if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestReference, digest); err != nil { + log.Error("Error setting package version property: %v", err) + return nil, err + } + } + + return pv, nil +} + +type blobReference struct { + Digest oci.Digest + MediaType oci.MediaType + Name string + File *packages_model.PackageFileDescriptor + ExpectedSize int64 + IsLead bool +} + +func createFileFromBlobReference(ctx context.Context, pv, uploadVersion *packages_model.PackageVersion, ref *blobReference) error { + if ref.File.Blob.Size != ref.ExpectedSize { + return errSizeInvalid + } + + if ref.Name == "" { + ref.Name = strings.ToLower(fmt.Sprintf("sha256_%s", ref.File.Blob.HashSHA256)) + } + + pf := &packages_model.PackageFile{ + VersionID: pv.ID, + BlobID: ref.File.Blob.ID, + Name: ref.Name, + LowerName: ref.Name, + IsLead: ref.IsLead, + } + var err error + if pf, err = packages_model.TryInsertFile(ctx, pf); err != nil { + log.Error("Error inserting package file: %v", err) + return err + } + + props := map[string]string{ + container_module.PropertyMediaType: string(ref.MediaType), + container_module.PropertyDigest: string(ref.Digest), + } + for name, value := range props { + if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeFile, pf.ID, name, value); err != nil { + log.Error("Error setting package file property: %v", err) + return err + } + } + + // Remove the file from the blob upload version + if uploadVersion != nil && ref.File.File != nil && uploadVersion.ID == ref.File.File.VersionID { + if err := packages_service.DeletePackageFile(ctx, ref.File.File); err != nil { + return err + } + } + + return nil +} + +func createManifestBlob(ctx context.Context, mci *manifestCreationInfo, pv *packages_model.PackageVersion, buf *packages_module.HashedBuffer) (*packages_model.PackageBlob, bool, string, error) { + pb, exists, err := packages_model.GetOrInsertBlob(ctx, packages_service.NewPackageBlob(buf)) + if err != nil { + log.Error("Error inserting package blob: %v", err) + return nil, false, "", err + } + if !exists { + contentStore := packages_module.NewContentStore() + if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), buf, buf.Size()); err != nil { + log.Error("Error saving package blob in content store: %v", err) + return nil, false, "", err + } + } + + manifestDigest := digestFromHashSummer(buf) + err = createFileFromBlobReference(ctx, pv, nil, &blobReference{ + Digest: oci.Digest(manifestDigest), + MediaType: mci.MediaType, + Name: container_model.ManifestFilename, + File: &packages_model.PackageFileDescriptor{Blob: pb}, + ExpectedSize: pb.Size, + IsLead: true, + }) + + return pb, !exists, manifestDigest, err +} diff --git a/routers/api/packages/generic/generic.go b/routers/api/packages/generic/generic.go new file mode 100644 index 0000000000..d862f77259 --- /dev/null +++ b/routers/api/packages/generic/generic.go @@ -0,0 +1,166 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package generic + +import ( + "errors" + "net/http" + "regexp" + + packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + packages_module "code.gitea.io/gitea/modules/packages" + "code.gitea.io/gitea/routers/api/packages/helper" + packages_service "code.gitea.io/gitea/services/packages" + + "github.com/hashicorp/go-version" +) + +var ( + packageNameRegex = regexp.MustCompile(`\A[A-Za-z0-9\.\_\-\+]+\z`) + filenameRegex = packageNameRegex +) + +func apiError(ctx *context.Context, status int, obj interface{}) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { + ctx.PlainText(status, message) + }) +} + +// DownloadPackageFile serves the specific generic package. +func DownloadPackageFile(ctx *context.Context) { + packageName, packageVersion, filename, err := sanitizeParameters(ctx) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + + s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + ctx, + &packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeGeneric, + Name: packageName, + Version: packageVersion, + }, + &packages_service.PackageFileInfo{ + Filename: filename, + }, + ) + if err != nil { + if err == packages_model.ErrPackageNotExist || err == packages_model.ErrPackageFileNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer s.Close() + + ctx.ServeStream(s, pf.Name) +} + +// UploadPackage uploads the specific generic package. +// Duplicated packages get rejected. +func UploadPackage(ctx *context.Context) { + packageName, packageVersion, filename, err := sanitizeParameters(ctx) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + + upload, close, err := ctx.UploadStream() + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if close { + defer upload.Close() + } + + buf, err := packages_module.CreateHashedBufferFromReader(upload, 32*1024*1024) + if err != nil { + log.Error("Error creating hashed buffer: %v", err) + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + _, _, err = packages_service.CreatePackageAndAddFile( + &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeGeneric, + Name: packageName, + Version: packageVersion, + }, + SemverCompatible: true, + Creator: ctx.Doer, + }, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: filename, + }, + Data: buf, + IsLead: true, + }, + ) + if err != nil { + if err == packages_model.ErrDuplicatePackageVersion { + apiError(ctx, http.StatusBadRequest, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.Status(http.StatusCreated) +} + +// DeletePackage deletes the specific generic package. +func DeletePackage(ctx *context.Context) { + packageName, packageVersion, _, err := sanitizeParameters(ctx) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + + err = packages_service.RemovePackageVersionByNameAndVersion( + ctx.Doer, + &packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeGeneric, + Name: packageName, + Version: packageVersion, + }, + ) + if err != nil { + if err == packages_model.ErrPackageNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.Status(http.StatusOK) +} + +func sanitizeParameters(ctx *context.Context) (string, string, string, error) { + packageName := ctx.Params("packagename") + filename := ctx.Params("filename") + + if !packageNameRegex.MatchString(packageName) || !filenameRegex.MatchString(filename) { + return "", "", "", errors.New("Invalid package name or filename") + } + + v, err := version.NewSemver(ctx.Params("packageversion")) + if err != nil { + return "", "", "", err + } + + return packageName, v.String(), filename, nil +} diff --git a/routers/api/packages/helper/helper.go b/routers/api/packages/helper/helper.go new file mode 100644 index 0000000000..8cde84023f --- /dev/null +++ b/routers/api/packages/helper/helper.go @@ -0,0 +1,38 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package helper + +import ( + "fmt" + "net/http" + + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" +) + +// LogAndProcessError logs an error and calls a custom callback with the processed error message. +// If the error is an InternalServerError the message is stripped if the user is not an admin. +func LogAndProcessError(ctx *context.Context, status int, obj interface{}, cb func(string)) { + var message string + if err, ok := obj.(error); ok { + message = err.Error() + } else if obj != nil { + message = fmt.Sprintf("%s", obj) + } + if status == http.StatusInternalServerError { + log.ErrorWithSkip(1, message) + + if setting.IsProd && (ctx.Doer == nil || !ctx.Doer.IsAdmin) { + message = "" + } + } else { + log.Debug(message) + } + + if cb != nil { + cb(message) + } +} diff --git a/routers/api/packages/maven/api.go b/routers/api/packages/maven/api.go new file mode 100644 index 0000000000..b60a317814 --- /dev/null +++ b/routers/api/packages/maven/api.go @@ -0,0 +1,56 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package maven + +import ( + "encoding/xml" + "sort" + "strings" + + packages_model "code.gitea.io/gitea/models/packages" + maven_module "code.gitea.io/gitea/modules/packages/maven" +) + +// MetadataResponse https://maven.apache.org/ref/3.2.5/maven-repository-metadata/repository-metadata.html +type MetadataResponse struct { + XMLName xml.Name `xml:"metadata"` + GroupID string `xml:"groupId"` + ArtifactID string `xml:"artifactId"` + Release string `xml:"versioning>release,omitempty"` + Latest string `xml:"versioning>latest"` + Version []string `xml:"versioning>versions>version"` +} + +func createMetadataResponse(pds []*packages_model.PackageDescriptor) *MetadataResponse { + sort.Slice(pds, func(i, j int) bool { + // Maven and Gradle order packages by their creation timestamp and not by their version string + return pds[i].Version.CreatedUnix < pds[j].Version.CreatedUnix + }) + + var release *packages_model.PackageDescriptor + + versions := make([]string, 0, len(pds)) + for _, pd := range pds { + if !strings.HasSuffix(pd.Version.Version, "-SNAPSHOT") { + release = pd + } + versions = append(versions, pd.Version.Version) + } + + latest := pds[len(pds)-1] + + metadata := latest.Metadata.(*maven_module.Metadata) + + resp := &MetadataResponse{ + GroupID: metadata.GroupID, + ArtifactID: metadata.ArtifactID, + Latest: latest.Version.Version, + Version: versions, + } + if release != nil { + resp.Release = release.Version.Version + } + return resp +} diff --git a/routers/api/packages/maven/maven.go b/routers/api/packages/maven/maven.go new file mode 100644 index 0000000000..bba4babf04 --- /dev/null +++ b/routers/api/packages/maven/maven.go @@ -0,0 +1,378 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package maven + +import ( + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "path/filepath" + "regexp" + "strings" + + packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + packages_module "code.gitea.io/gitea/modules/packages" + maven_module "code.gitea.io/gitea/modules/packages/maven" + "code.gitea.io/gitea/routers/api/packages/helper" + packages_service "code.gitea.io/gitea/services/packages" +) + +const ( + mavenMetadataFile = "maven-metadata.xml" + extensionMD5 = ".md5" + extensionSHA1 = ".sha1" + extensionSHA256 = ".sha256" + extensionSHA512 = ".sha512" +) + +var ( + errInvalidParameters = errors.New("request parameters are invalid") + illegalCharacters = regexp.MustCompile(`[\\/:"<>|?\*]`) +) + +func apiError(ctx *context.Context, status int, obj interface{}) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { + ctx.PlainText(status, message) + }) +} + +// DownloadPackageFile serves the content of a package +func DownloadPackageFile(ctx *context.Context) { + params, err := extractPathParameters(ctx) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + + if params.IsMeta && params.Version == "" { + serveMavenMetadata(ctx, params) + } else { + servePackageFile(ctx, params) + } +} + +func serveMavenMetadata(ctx *context.Context, params parameters) { + // /com/foo/project/maven-metadata.xml[.md5/.sha1/.sha256/.sha512] + + packageName := params.GroupID + "-" + params.ArtifactID + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeMaven, packageName) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pvs) == 0 { + apiError(ctx, http.StatusNotFound, packages_model.ErrPackageNotExist) + return + } + + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + xmlMetadata, err := xml.Marshal(createMetadataResponse(pds)) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + xmlMetadataWithHeader := append([]byte(xml.Header), xmlMetadata...) + + ext := strings.ToLower(filepath.Ext(params.Filename)) + if isChecksumExtension(ext) { + var hash []byte + switch ext { + case extensionMD5: + tmp := md5.Sum(xmlMetadataWithHeader) + hash = tmp[:] + case extensionSHA1: + tmp := sha1.Sum(xmlMetadataWithHeader) + hash = tmp[:] + case extensionSHA256: + tmp := sha256.Sum256(xmlMetadataWithHeader) + hash = tmp[:] + case extensionSHA512: + tmp := sha512.Sum512(xmlMetadataWithHeader) + hash = tmp[:] + } + ctx.PlainText(http.StatusOK, fmt.Sprintf("%x", hash)) + return + } + + ctx.PlainTextBytes(http.StatusOK, xmlMetadataWithHeader) +} + +func servePackageFile(ctx *context.Context, params parameters) { + packageName := params.GroupID + "-" + params.ArtifactID + + pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeMaven, packageName, params.Version) + if err != nil { + if err == packages_model.ErrPackageNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + filename := params.Filename + + ext := strings.ToLower(filepath.Ext(filename)) + if isChecksumExtension(ext) { + filename = filename[:len(filename)-len(ext)] + } + + pf, err := packages_model.GetFileForVersionByName(ctx, pv.ID, filename, packages_model.EmptyFileKey) + if err != nil { + if err == packages_model.ErrPackageFileNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + pb, err := packages_model.GetBlobByID(ctx, pf.BlobID) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if isChecksumExtension(ext) { + var hash string + switch ext { + case extensionMD5: + hash = pb.HashMD5 + case extensionSHA1: + hash = pb.HashSHA1 + case extensionSHA256: + hash = pb.HashSHA256 + case extensionSHA512: + hash = pb.HashSHA512 + } + ctx.PlainText(http.StatusOK, hash) + return + } + + s, err := packages_module.NewContentStore().Get(packages_module.BlobHash256Key(pb.HashSHA256)) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + } + defer s.Close() + + if pf.IsLead { + if err := packages_model.IncrementDownloadCounter(ctx, pv.ID); err != nil { + log.Error("Error incrementing download counter: %v", err) + } + } + + ctx.ServeStream(s, pf.Name) +} + +// UploadPackageFile adds a file to the package. If the package does not exist, it gets created. +func UploadPackageFile(ctx *context.Context) { + params, err := extractPathParameters(ctx) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + + log.Trace("Parameters: %+v", params) + + // Ignore the package index /<name>/maven-metadata.xml + if params.IsMeta && params.Version == "" { + ctx.Status(http.StatusOK) + return + } + + packageName := params.GroupID + "-" + params.ArtifactID + + buf, err := packages_module.CreateHashedBufferFromReader(ctx.Req.Body, 32*1024*1024) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + pvci := &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeMaven, + Name: packageName, + Version: params.Version, + }, + SemverCompatible: false, + Creator: ctx.Doer, + } + + ext := filepath.Ext(params.Filename) + + // Do not upload checksum files but compare the hashes. + if isChecksumExtension(ext) { + pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvci.Owner.ID, pvci.PackageType, pvci.Name, pvci.Version) + if err != nil { + if err == packages_model.ErrPackageNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + pf, err := packages_model.GetFileForVersionByName(ctx, pv.ID, params.Filename[:len(params.Filename)-len(ext)], packages_model.EmptyFileKey) + if err != nil { + if err == packages_model.ErrPackageFileNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + pb, err := packages_model.GetBlobByID(ctx, pf.BlobID) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + hash, err := io.ReadAll(buf) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if (ext == extensionMD5 && pb.HashMD5 != string(hash)) || + (ext == extensionSHA1 && pb.HashSHA1 != string(hash)) || + (ext == extensionSHA256 && pb.HashSHA256 != string(hash)) || + (ext == extensionSHA512 && pb.HashSHA512 != string(hash)) { + apiError(ctx, http.StatusBadRequest, "hash mismatch") + return + } + + ctx.Status(http.StatusOK) + return + } + + pfci := &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: params.Filename, + }, + Data: buf, + IsLead: false, + } + + // If it's the package pom file extract the metadata + if ext == ".pom" { + pfci.IsLead = true + + var err error + pvci.Metadata, err = maven_module.ParsePackageMetaData(buf) + if err != nil { + log.Error("Error parsing package metadata: %v", err) + } + + if pvci.Metadata != nil { + pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvci.Owner.ID, pvci.PackageType, pvci.Name, pvci.Version) + if err != nil && err != packages_model.ErrPackageNotExist { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if pv != nil { + raw, err := json.Marshal(pvci.Metadata) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + pv.MetadataJSON = string(raw) + if err := packages_model.UpdateVersion(ctx, pv); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + } + } + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + } + + _, _, err = packages_service.CreatePackageOrAddFileToExisting( + pvci, + pfci, + ) + if err != nil { + if err == packages_model.ErrDuplicatePackageFile { + apiError(ctx, http.StatusBadRequest, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.Status(http.StatusCreated) +} + +func isChecksumExtension(ext string) bool { + return ext == extensionMD5 || ext == extensionSHA1 || ext == extensionSHA256 || ext == extensionSHA512 +} + +type parameters struct { + GroupID string + ArtifactID string + Version string + Filename string + IsMeta bool +} + +func extractPathParameters(ctx *context.Context) (parameters, error) { + parts := strings.Split(ctx.Params("*"), "/") + + p := parameters{ + Filename: parts[len(parts)-1], + } + + p.IsMeta = p.Filename == mavenMetadataFile || + p.Filename == mavenMetadataFile+extensionMD5 || + p.Filename == mavenMetadataFile+extensionSHA1 || + p.Filename == mavenMetadataFile+extensionSHA256 || + p.Filename == mavenMetadataFile+extensionSHA512 + + parts = parts[:len(parts)-1] + if len(parts) == 0 { + return p, errInvalidParameters + } + + p.Version = parts[len(parts)-1] + if p.IsMeta && !strings.HasSuffix(p.Version, "-SNAPSHOT") { + p.Version = "" + } else { + parts = parts[:len(parts)-1] + } + + if illegalCharacters.MatchString(p.Version) { + return p, errInvalidParameters + } + + if len(parts) < 2 { + return p, errInvalidParameters + } + + p.ArtifactID = parts[len(parts)-1] + p.GroupID = strings.Join(parts[:len(parts)-1], ".") + + if illegalCharacters.MatchString(p.GroupID) || illegalCharacters.MatchString(p.ArtifactID) { + return p, errInvalidParameters + } + + return p, nil +} diff --git a/routers/api/packages/npm/api.go b/routers/api/packages/npm/api.go new file mode 100644 index 0000000000..56c8977043 --- /dev/null +++ b/routers/api/packages/npm/api.go @@ -0,0 +1,73 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package npm + +import ( + "encoding/base64" + "encoding/hex" + "fmt" + "net/url" + "sort" + + packages_model "code.gitea.io/gitea/models/packages" + npm_module "code.gitea.io/gitea/modules/packages/npm" +) + +func createPackageMetadataResponse(registryURL string, pds []*packages_model.PackageDescriptor) *npm_module.PackageMetadata { + sort.Slice(pds, func(i, j int) bool { + return pds[i].SemVer.LessThan(pds[j].SemVer) + }) + + versions := make(map[string]*npm_module.PackageMetadataVersion) + distTags := make(map[string]string) + for _, pd := range pds { + versions[pd.SemVer.String()] = createPackageMetadataVersion(registryURL, pd) + + for _, pvp := range pd.Properties { + if pvp.Name == npm_module.TagProperty { + distTags[pvp.Value] = pd.Version.Version + } + } + } + + latest := pds[len(pds)-1] + + metadata := latest.Metadata.(*npm_module.Metadata) + + return &npm_module.PackageMetadata{ + ID: latest.Package.Name, + Name: latest.Package.Name, + DistTags: distTags, + Description: metadata.Description, + Readme: metadata.Readme, + Homepage: metadata.ProjectURL, + Author: npm_module.User{Name: metadata.Author}, + License: metadata.License, + Versions: versions, + } +} + +func createPackageMetadataVersion(registryURL string, pd *packages_model.PackageDescriptor) *npm_module.PackageMetadataVersion { + hashBytes, _ := hex.DecodeString(pd.Files[0].Blob.HashSHA512) + + metadata := pd.Metadata.(*npm_module.Metadata) + + return &npm_module.PackageMetadataVersion{ + ID: fmt.Sprintf("%s@%s", pd.Package.Name, pd.Version.Version), + Name: pd.Package.Name, + Version: pd.Version.Version, + Description: metadata.Description, + Author: npm_module.User{Name: metadata.Author}, + Homepage: metadata.ProjectURL, + License: metadata.License, + Dependencies: metadata.Dependencies, + Readme: metadata.Readme, + Dist: npm_module.PackageDistribution{ + Shasum: pd.Files[0].Blob.HashSHA1, + Integrity: "sha512-" + base64.StdEncoding.EncodeToString(hashBytes), + Tarball: fmt.Sprintf("%s/%s/-/%s/%s", registryURL, url.QueryEscape(pd.Package.Name), url.PathEscape(pd.Version.Version), url.PathEscape(pd.Files[0].File.LowerName)), + }, + } +} diff --git a/routers/api/packages/npm/npm.go b/routers/api/packages/npm/npm.go new file mode 100644 index 0000000000..50151ee5ea --- /dev/null +++ b/routers/api/packages/npm/npm.go @@ -0,0 +1,288 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package npm + +import ( + "bytes" + "errors" + "fmt" + "io" + "net/http" + "strings" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/context" + packages_module "code.gitea.io/gitea/modules/packages" + npm_module "code.gitea.io/gitea/modules/packages/npm" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/routers/api/packages/helper" + packages_service "code.gitea.io/gitea/services/packages" + + "github.com/hashicorp/go-version" +) + +// errInvalidTagName indicates an invalid tag name +var errInvalidTagName = errors.New("The tag name is invalid") + +func apiError(ctx *context.Context, status int, obj interface{}) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { + ctx.JSON(status, map[string]string{ + "error": message, + }) + }) +} + +// packageNameFromParams gets the package name from the url parameters +// Variations: /name/, /@scope/name/, /@scope%2Fname/ +func packageNameFromParams(ctx *context.Context) string { + scope := ctx.Params("scope") + id := ctx.Params("id") + if scope != "" { + return fmt.Sprintf("@%s/%s", scope, id) + } + return id +} + +// PackageMetadata returns the metadata for a single package +func PackageMetadata(ctx *context.Context) { + packageName := packageNameFromParams(ctx) + + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeNpm, packageName) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pvs) == 0 { + apiError(ctx, http.StatusNotFound, err) + return + } + + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + resp := createPackageMetadataResponse( + setting.AppURL+"api/packages/"+ctx.Package.Owner.Name+"/npm", + pds, + ) + + ctx.JSON(http.StatusOK, resp) +} + +// DownloadPackageFile serves the content of a package +func DownloadPackageFile(ctx *context.Context) { + packageName := packageNameFromParams(ctx) + packageVersion := ctx.Params("version") + filename := ctx.Params("filename") + + s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + ctx, + &packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeNpm, + Name: packageName, + Version: packageVersion, + }, + &packages_service.PackageFileInfo{ + Filename: filename, + }, + ) + if err != nil { + if err == packages_model.ErrPackageNotExist || err == packages_model.ErrPackageFileNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer s.Close() + + ctx.ServeStream(s, pf.Name) +} + +// UploadPackage creates a new package +func UploadPackage(ctx *context.Context) { + npmPackage, err := npm_module.ParsePackage(ctx.Req.Body) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + + buf, err := packages_module.CreateHashedBufferFromReader(bytes.NewReader(npmPackage.Data), 32*1024*1024) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + pv, _, err := packages_service.CreatePackageAndAddFile( + &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeNpm, + Name: npmPackage.Name, + Version: npmPackage.Version, + }, + SemverCompatible: true, + Creator: ctx.Doer, + Metadata: npmPackage.Metadata, + }, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: npmPackage.Filename, + }, + Data: buf, + IsLead: true, + }, + ) + if err != nil { + if err == packages_model.ErrDuplicatePackageVersion { + apiError(ctx, http.StatusBadRequest, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + + for _, tag := range npmPackage.DistTags { + if err := setPackageTag(tag, pv, false); err != nil { + if err == errInvalidTagName { + apiError(ctx, http.StatusBadRequest, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + } + + ctx.Status(http.StatusCreated) +} + +// ListPackageTags returns all tags for a package +func ListPackageTags(ctx *context.Context) { + packageName := packageNameFromParams(ctx) + + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeNpm, packageName) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + tags := make(map[string]string) + for _, pv := range pvs { + pvps, err := packages_model.GetPropertiesByName(ctx, packages_model.PropertyTypeVersion, pv.ID, npm_module.TagProperty) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + for _, pvp := range pvps { + tags[pvp.Value] = pv.Version + } + } + + ctx.JSON(http.StatusOK, tags) +} + +// AddPackageTag adds a tag to the package +func AddPackageTag(ctx *context.Context) { + packageName := packageNameFromParams(ctx) + + body, err := io.ReadAll(ctx.Req.Body) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + version := strings.Trim(string(body), "\"") // is as "version" in the body + + pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeNpm, packageName, version) + if err != nil { + if err == packages_model.ErrPackageNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if err := setPackageTag(ctx.Params("tag"), pv, false); err != nil { + if err == errInvalidTagName { + apiError(ctx, http.StatusBadRequest, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } +} + +// DeletePackageTag deletes a package tag +func DeletePackageTag(ctx *context.Context) { + packageName := packageNameFromParams(ctx) + + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeNpm, packageName) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if len(pvs) != 0 { + if err := setPackageTag(ctx.Params("tag"), pvs[0], true); err != nil { + if err == errInvalidTagName { + apiError(ctx, http.StatusBadRequest, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + } +} + +func setPackageTag(tag string, pv *packages_model.PackageVersion, deleteOnly bool) error { + if tag == "" { + return errInvalidTagName + } + _, err := version.NewVersion(tag) + if err == nil { + return errInvalidTagName + } + + ctx, committer, err := db.TxContext() + if err != nil { + return err + } + defer committer.Close() + + pvs, err := packages_model.FindVersionsByPropertyNameAndValue(ctx, pv.PackageID, npm_module.TagProperty, tag) + if err != nil { + return err + } + + if len(pvs) == 1 { + pvps, err := packages_model.GetPropertiesByName(ctx, packages_model.PropertyTypeVersion, pvs[0].ID, npm_module.TagProperty) + if err != nil { + return err + } + + for _, pvp := range pvps { + if pvp.Value == tag { + if err := packages_model.DeletePropertyByID(ctx, pvp.ID); err != nil { + return err + } + break + } + } + } + + if !deleteOnly { + _, err = packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, npm_module.TagProperty, tag) + if err != nil { + return err + } + } + + return committer.Commit() +} diff --git a/routers/api/packages/nuget/api.go b/routers/api/packages/nuget/api.go new file mode 100644 index 0000000000..b449cfc5bb --- /dev/null +++ b/routers/api/packages/nuget/api.go @@ -0,0 +1,287 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package nuget + +import ( + "bytes" + "fmt" + "sort" + "time" + + packages_model "code.gitea.io/gitea/models/packages" + nuget_module "code.gitea.io/gitea/modules/packages/nuget" + + "github.com/hashicorp/go-version" +) + +// ServiceIndexResponse https://docs.microsoft.com/en-us/nuget/api/service-index#resources +type ServiceIndexResponse struct { + Version string `json:"version"` + Resources []ServiceResource `json:"resources"` +} + +// ServiceResource https://docs.microsoft.com/en-us/nuget/api/service-index#resource +type ServiceResource struct { + ID string `json:"@id"` + Type string `json:"@type"` +} + +func createServiceIndexResponse(root string) *ServiceIndexResponse { + return &ServiceIndexResponse{ + Version: "3.0.0", + Resources: []ServiceResource{ + {ID: root + "/query", Type: "SearchQueryService"}, + {ID: root + "/query", Type: "SearchQueryService/3.0.0-beta"}, + {ID: root + "/query", Type: "SearchQueryService/3.0.0-rc"}, + {ID: root + "/registration", Type: "RegistrationsBaseUrl"}, + {ID: root + "/registration", Type: "RegistrationsBaseUrl/3.0.0-beta"}, + {ID: root + "/registration", Type: "RegistrationsBaseUrl/3.0.0-rc"}, + {ID: root + "/package", Type: "PackageBaseAddress/3.0.0"}, + {ID: root, Type: "PackagePublish/2.0.0"}, + {ID: root + "/symbolpackage", Type: "SymbolPackagePublish/4.9.0"}, + }, + } +} + +// RegistrationIndexResponse https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#response +type RegistrationIndexResponse struct { + RegistrationIndexURL string `json:"@id"` + Type []string `json:"@type"` + Count int `json:"count"` + Pages []*RegistrationIndexPage `json:"items"` +} + +// RegistrationIndexPage https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#registration-page-object +type RegistrationIndexPage struct { + RegistrationPageURL string `json:"@id"` + Lower string `json:"lower"` + Upper string `json:"upper"` + Count int `json:"count"` + Items []*RegistrationIndexPageItem `json:"items"` +} + +// RegistrationIndexPageItem https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#registration-leaf-object-in-a-page +type RegistrationIndexPageItem struct { + RegistrationLeafURL string `json:"@id"` + PackageContentURL string `json:"packageContent"` + CatalogEntry *CatalogEntry `json:"catalogEntry"` +} + +// CatalogEntry https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#catalog-entry +type CatalogEntry struct { + CatalogLeafURL string `json:"@id"` + PackageContentURL string `json:"packageContent"` + ID string `json:"id"` + Version string `json:"version"` + Description string `json:"description"` + ReleaseNotes string `json:"releaseNotes"` + Authors string `json:"authors"` + RequireLicenseAcceptance bool `json:"requireLicenseAcceptance"` + ProjectURL string `json:"projectURL"` + DependencyGroups []*PackageDependencyGroup `json:"dependencyGroups"` +} + +// PackageDependencyGroup https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#package-dependency-group +type PackageDependencyGroup struct { + TargetFramework string `json:"targetFramework"` + Dependencies []*PackageDependency `json:"dependencies"` +} + +// PackageDependency https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#package-dependency +type PackageDependency struct { + ID string `json:"id"` + Range string `json:"range"` +} + +func createRegistrationIndexResponse(l *linkBuilder, pds []*packages_model.PackageDescriptor) *RegistrationIndexResponse { + sort.Slice(pds, func(i, j int) bool { + return pds[i].SemVer.LessThan(pds[j].SemVer) + }) + + items := make([]*RegistrationIndexPageItem, 0, len(pds)) + for _, p := range pds { + items = append(items, createRegistrationIndexPageItem(l, p)) + } + + return &RegistrationIndexResponse{ + RegistrationIndexURL: l.GetRegistrationIndexURL(pds[0].Package.Name), + Type: []string{"catalog:CatalogRoot", "PackageRegistration", "catalog:Permalink"}, + Count: 1, + Pages: []*RegistrationIndexPage{ + { + RegistrationPageURL: l.GetRegistrationIndexURL(pds[0].Package.Name), + Count: len(pds), + Lower: normalizeVersion(pds[0].SemVer), + Upper: normalizeVersion(pds[len(pds)-1].SemVer), + Items: items, + }, + }, + } +} + +func createRegistrationIndexPageItem(l *linkBuilder, pd *packages_model.PackageDescriptor) *RegistrationIndexPageItem { + metadata := pd.Metadata.(*nuget_module.Metadata) + + return &RegistrationIndexPageItem{ + RegistrationLeafURL: l.GetRegistrationLeafURL(pd.Package.Name, pd.Version.Version), + PackageContentURL: l.GetPackageDownloadURL(pd.Package.Name, pd.Version.Version), + CatalogEntry: &CatalogEntry{ + CatalogLeafURL: l.GetRegistrationLeafURL(pd.Package.Name, pd.Version.Version), + PackageContentURL: l.GetPackageDownloadURL(pd.Package.Name, pd.Version.Version), + ID: pd.Package.Name, + Version: pd.Version.Version, + Description: metadata.Description, + ReleaseNotes: metadata.ReleaseNotes, + Authors: metadata.Authors, + ProjectURL: metadata.ProjectURL, + DependencyGroups: createDependencyGroups(pd), + }, + } +} + +func createDependencyGroups(pd *packages_model.PackageDescriptor) []*PackageDependencyGroup { + metadata := pd.Metadata.(*nuget_module.Metadata) + + dependencyGroups := make([]*PackageDependencyGroup, 0, len(metadata.Dependencies)) + for k, v := range metadata.Dependencies { + dependencies := make([]*PackageDependency, 0, len(v)) + for _, dep := range v { + dependencies = append(dependencies, &PackageDependency{ + ID: dep.ID, + Range: dep.Version, + }) + } + + dependencyGroups = append(dependencyGroups, &PackageDependencyGroup{ + TargetFramework: k, + Dependencies: dependencies, + }) + } + return dependencyGroups +} + +// RegistrationLeafResponse https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#registration-leaf +type RegistrationLeafResponse struct { + RegistrationLeafURL string `json:"@id"` + Type []string `json:"@type"` + Listed bool `json:"listed"` + PackageContentURL string `json:"packageContent"` + Published time.Time `json:"published"` + RegistrationIndexURL string `json:"registration"` +} + +func createRegistrationLeafResponse(l *linkBuilder, pd *packages_model.PackageDescriptor) *RegistrationLeafResponse { + return &RegistrationLeafResponse{ + Type: []string{"Package", "http://schema.nuget.org/catalog#Permalink"}, + Listed: true, + Published: time.Unix(int64(pd.Version.CreatedUnix), 0), + RegistrationLeafURL: l.GetRegistrationLeafURL(pd.Package.Name, pd.Version.Version), + PackageContentURL: l.GetPackageDownloadURL(pd.Package.Name, pd.Version.Version), + RegistrationIndexURL: l.GetRegistrationIndexURL(pd.Package.Name), + } +} + +// PackageVersionsResponse https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource#response +type PackageVersionsResponse struct { + Versions []string `json:"versions"` +} + +func createPackageVersionsResponse(pds []*packages_model.PackageDescriptor) *PackageVersionsResponse { + versions := make([]string, 0, len(pds)) + for _, pd := range pds { + versions = append(versions, normalizeVersion(pd.SemVer)) + } + + return &PackageVersionsResponse{ + Versions: versions, + } +} + +// SearchResultResponse https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#response +type SearchResultResponse struct { + TotalHits int64 `json:"totalHits"` + Data []*SearchResult `json:"data"` +} + +// SearchResult https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#search-result +type SearchResult struct { + ID string `json:"id"` + Version string `json:"version"` + Versions []*SearchResultVersion `json:"versions"` + Description string `json:"description"` + Authors string `json:"authors"` + ProjectURL string `json:"projectURL"` + RegistrationIndexURL string `json:"registration"` +} + +// SearchResultVersion https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#search-result +type SearchResultVersion struct { + RegistrationLeafURL string `json:"@id"` + Version string `json:"version"` + Downloads int64 `json:"downloads"` +} + +func createSearchResultResponse(l *linkBuilder, totalHits int64, pds []*packages_model.PackageDescriptor) *SearchResultResponse { + data := make([]*SearchResult, 0, len(pds)) + + if len(pds) > 0 { + groupID := pds[0].Package.Name + group := make([]*packages_model.PackageDescriptor, 0, 10) + + for i := 0; i < len(pds); i++ { + if groupID != pds[i].Package.Name { + data = append(data, createSearchResult(l, group)) + groupID = pds[i].Package.Name + group = group[:0] + } + group = append(group, pds[i]) + } + data = append(data, createSearchResult(l, group)) + } + + return &SearchResultResponse{ + TotalHits: totalHits, + Data: data, + } +} + +func createSearchResult(l *linkBuilder, pds []*packages_model.PackageDescriptor) *SearchResult { + latest := pds[0] + versions := make([]*SearchResultVersion, 0, len(pds)) + for _, pd := range pds { + if latest.SemVer.LessThan(pd.SemVer) { + latest = pd + } + + versions = append(versions, &SearchResultVersion{ + RegistrationLeafURL: l.GetRegistrationLeafURL(pd.Package.Name, pd.Version.Version), + Version: pd.Version.Version, + }) + } + + metadata := latest.Metadata.(*nuget_module.Metadata) + + return &SearchResult{ + ID: latest.Package.Name, + Version: latest.Version.Version, + Versions: versions, + Description: metadata.Description, + Authors: metadata.Authors, + ProjectURL: metadata.ProjectURL, + RegistrationIndexURL: l.GetRegistrationIndexURL(latest.Package.Name), + } +} + +// normalizeVersion removes the metadata +func normalizeVersion(v *version.Version) string { + var buf bytes.Buffer + segments := v.Segments64() + fmt.Fprintf(&buf, "%d.%d.%d", segments[0], segments[1], segments[2]) + pre := v.Prerelease() + if pre != "" { + fmt.Fprintf(&buf, "-%s", pre) + } + return buf.String() +} diff --git a/routers/api/packages/nuget/links.go b/routers/api/packages/nuget/links.go new file mode 100644 index 0000000000..f782c7f2cb --- /dev/null +++ b/routers/api/packages/nuget/links.go @@ -0,0 +1,28 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package nuget + +import ( + "fmt" +) + +type linkBuilder struct { + Base string +} + +// GetRegistrationIndexURL builds the registration index url +func (l *linkBuilder) GetRegistrationIndexURL(id string) string { + return fmt.Sprintf("%s/registration/%s/index.json", l.Base, id) +} + +// GetRegistrationLeafURL builds the registration leaf url +func (l *linkBuilder) GetRegistrationLeafURL(id, version string) string { + return fmt.Sprintf("%s/registration/%s/%s.json", l.Base, id, version) +} + +// GetPackageDownloadURL builds the download url +func (l *linkBuilder) GetPackageDownloadURL(id, version string) string { + return fmt.Sprintf("%s/package/%s/%s/%s.%s.nupkg", l.Base, id, version, id, version) +} diff --git a/routers/api/packages/nuget/nuget.go b/routers/api/packages/nuget/nuget.go new file mode 100644 index 0000000000..f3bc586125 --- /dev/null +++ b/routers/api/packages/nuget/nuget.go @@ -0,0 +1,421 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package nuget + +import ( + "errors" + "fmt" + "io" + "net/http" + "strings" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/context" + packages_module "code.gitea.io/gitea/modules/packages" + nuget_module "code.gitea.io/gitea/modules/packages/nuget" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/routers/api/packages/helper" + packages_service "code.gitea.io/gitea/services/packages" +) + +func apiError(ctx *context.Context, status int, obj interface{}) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { + ctx.JSON(status, map[string]string{ + "Message": message, + }) + }) +} + +// ServiceIndex https://docs.microsoft.com/en-us/nuget/api/service-index +func ServiceIndex(ctx *context.Context) { + resp := createServiceIndexResponse(setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/nuget") + + ctx.JSON(http.StatusOK, resp) +} + +// SearchService https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#search-for-packages +func SearchService(ctx *context.Context) { + pvs, count, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + Type: string(packages_model.TypeNuGet), + QueryName: ctx.FormTrim("q"), + Paginator: db.NewAbsoluteListOptions( + ctx.FormInt("skip"), + ctx.FormInt("take"), + ), + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + resp := createSearchResultResponse( + &linkBuilder{setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/nuget"}, + count, + pds, + ) + + ctx.JSON(http.StatusOK, resp) +} + +// RegistrationIndex https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#registration-index +func RegistrationIndex(ctx *context.Context) { + packageName := ctx.Params("id") + + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeNuGet, packageName) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pvs) == 0 { + apiError(ctx, http.StatusNotFound, err) + return + } + + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + resp := createRegistrationIndexResponse( + &linkBuilder{setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/nuget"}, + pds, + ) + + ctx.JSON(http.StatusOK, resp) +} + +// RegistrationLeaf https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#registration-leaf +func RegistrationLeaf(ctx *context.Context) { + packageName := ctx.Params("id") + packageVersion := strings.TrimSuffix(ctx.Params("version"), ".json") + + pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, ctx.Package.Owner.ID, packages_model.TypeNuGet, packageName, packageVersion) + if err != nil { + if err == packages_model.ErrPackageNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + + pd, err := packages_model.GetPackageDescriptor(ctx, pv) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + resp := createRegistrationLeafResponse( + &linkBuilder{setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/nuget"}, + pd, + ) + + ctx.JSON(http.StatusOK, resp) +} + +// EnumeratePackageVersions https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource#enumerate-package-versions +func EnumeratePackageVersions(ctx *context.Context) { + packageName := ctx.Params("id") + + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeNuGet, packageName) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pvs) == 0 { + apiError(ctx, http.StatusNotFound, err) + return + } + + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + resp := createPackageVersionsResponse(pds) + + ctx.JSON(http.StatusOK, resp) +} + +// DownloadPackageFile https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource#download-package-content-nupkg +func DownloadPackageFile(ctx *context.Context) { + packageName := ctx.Params("id") + packageVersion := ctx.Params("version") + filename := ctx.Params("filename") + + s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + ctx, + &packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeNuGet, + Name: packageName, + Version: packageVersion, + }, + &packages_service.PackageFileInfo{ + Filename: filename, + }, + ) + if err != nil { + if err == packages_model.ErrPackageNotExist || err == packages_model.ErrPackageFileNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer s.Close() + + ctx.ServeStream(s, pf.Name) +} + +// UploadPackage creates a new package with the metadata contained in the uploaded nupgk file +// https://docs.microsoft.com/en-us/nuget/api/package-publish-resource#push-a-package +func UploadPackage(ctx *context.Context) { + np, buf, closables := processUploadedFile(ctx, nuget_module.DependencyPackage) + defer func() { + for _, c := range closables { + c.Close() + } + }() + if np == nil { + return + } + + _, _, err := packages_service.CreatePackageAndAddFile( + &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeNuGet, + Name: np.ID, + Version: np.Version, + }, + SemverCompatible: true, + Creator: ctx.Doer, + Metadata: np.Metadata, + }, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: strings.ToLower(fmt.Sprintf("%s.%s.nupkg", np.ID, np.Version)), + }, + Data: buf, + IsLead: true, + }, + ) + if err != nil { + if err == packages_model.ErrDuplicatePackageVersion { + apiError(ctx, http.StatusBadRequest, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.Status(http.StatusCreated) +} + +// UploadSymbolPackage adds a symbol package to an existing package +// https://docs.microsoft.com/en-us/nuget/api/symbol-package-publish-resource +func UploadSymbolPackage(ctx *context.Context) { + np, buf, closables := processUploadedFile(ctx, nuget_module.SymbolsPackage) + defer func() { + for _, c := range closables { + c.Close() + } + }() + if np == nil { + return + } + + pdbs, err := nuget_module.ExtractPortablePdb(buf, buf.Size()) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + defer pdbs.Close() + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + pi := &packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeNuGet, + Name: np.ID, + Version: np.Version, + } + + _, _, err = packages_service.AddFileToExistingPackage( + pi, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: strings.ToLower(fmt.Sprintf("%s.%s.snupkg", np.ID, np.Version)), + }, + Data: buf, + IsLead: false, + }, + ) + if err != nil { + switch err { + case packages_model.ErrPackageNotExist: + apiError(ctx, http.StatusNotFound, err) + case packages_model.ErrDuplicatePackageFile: + apiError(ctx, http.StatusBadRequest, err) + default: + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + for _, pdb := range pdbs { + _, _, err := packages_service.AddFileToExistingPackage( + pi, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: strings.ToLower(pdb.Name), + CompositeKey: strings.ToLower(pdb.ID), + }, + Data: pdb.Content, + IsLead: false, + Properties: map[string]string{ + nuget_module.PropertySymbolID: strings.ToLower(pdb.ID), + }, + }, + ) + if err != nil { + switch err { + case packages_model.ErrDuplicatePackageFile: + apiError(ctx, http.StatusBadRequest, err) + default: + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + } + + ctx.Status(http.StatusCreated) +} + +func processUploadedFile(ctx *context.Context, expectedType nuget_module.PackageType) (*nuget_module.Package, *packages_module.HashedBuffer, []io.Closer) { + closables := make([]io.Closer, 0, 2) + + upload, close, err := ctx.UploadStream() + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return nil, nil, closables + } + + if close { + closables = append(closables, upload) + } + + buf, err := packages_module.CreateHashedBufferFromReader(upload, 32*1024*1024) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return nil, nil, closables + } + closables = append(closables, buf) + + np, err := nuget_module.ParsePackageMetaData(buf, buf.Size()) + if err != nil { + if err == nuget_module.ErrMissingNuspecFile || err == nuget_module.ErrNuspecFileTooLarge || err == nuget_module.ErrNuspecInvalidID || err == nuget_module.ErrNuspecInvalidVersion { + apiError(ctx, http.StatusBadRequest, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return nil, nil, closables + } + if np.PackageType != expectedType { + apiError(ctx, http.StatusBadRequest, errors.New("unexpected package type")) + return nil, nil, closables + } + if _, err := buf.Seek(0, io.SeekStart); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return nil, nil, closables + } + return np, buf, closables +} + +// DownloadSymbolFile https://github.com/dotnet/symstore/blob/main/docs/specs/Simple_Symbol_Query_Protocol.md#request +func DownloadSymbolFile(ctx *context.Context) { + filename := ctx.Params("filename") + guid := ctx.Params("guid") + filename2 := ctx.Params("filename2") + + if filename != filename2 { + apiError(ctx, http.StatusBadRequest, nil) + return + } + + pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + PackageType: string(packages_model.TypeNuGet), + Query: filename, + Properties: map[string]string{ + nuget_module.PropertySymbolID: strings.ToLower(guid), + }, + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pfs) != 1 { + apiError(ctx, http.StatusNotFound, nil) + return + } + + pv, err := packages_model.GetVersionByID(ctx, pfs[0].VersionID) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + s, _, err := packages_service.GetPackageFileStream(ctx, pv, pfs[0]) + if err != nil { + if err == packages_model.ErrPackageNotExist || err == packages_model.ErrPackageFileNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer s.Close() + + ctx.ServeStream(s, pfs[0].Name) +} + +// DeletePackage hard deletes the package +// https://docs.microsoft.com/en-us/nuget/api/package-publish-resource#delete-a-package +func DeletePackage(ctx *context.Context) { + packageName := ctx.Params("id") + packageVersion := ctx.Params("version") + + err := packages_service.RemovePackageVersionByNameAndVersion( + ctx.Doer, + &packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeNuGet, + Name: packageName, + Version: packageVersion, + }, + ) + if err != nil { + if err == packages_model.ErrPackageNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + } +} diff --git a/routers/api/packages/pypi/pypi.go b/routers/api/packages/pypi/pypi.go new file mode 100644 index 0000000000..9209c4edd5 --- /dev/null +++ b/routers/api/packages/pypi/pypi.go @@ -0,0 +1,174 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package pypi + +import ( + "fmt" + "io" + "net/http" + "regexp" + "strings" + + packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/context" + packages_module "code.gitea.io/gitea/modules/packages" + pypi_module "code.gitea.io/gitea/modules/packages/pypi" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/templates" + "code.gitea.io/gitea/modules/validation" + "code.gitea.io/gitea/routers/api/packages/helper" + packages_service "code.gitea.io/gitea/services/packages" +) + +// https://www.python.org/dev/peps/pep-0503/#normalized-names +var normalizer = strings.NewReplacer(".", "-", "_", "-") +var nameMatcher = regexp.MustCompile(`\A[a-z0-9\.\-_]+\z`) + +// https://www.python.org/dev/peps/pep-0440/#appendix-b-parsing-version-strings-with-regular-expressions +var versionMatcher = regexp.MustCompile(`^([1-9][0-9]*!)?(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))*((a|b|rc)(0|[1-9][0-9]*))?(\.post(0|[1-9][0-9]*))?(\.dev(0|[1-9][0-9]*))?$`) + +func apiError(ctx *context.Context, status int, obj interface{}) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { + ctx.PlainText(status, message) + }) +} + +// PackageMetadata returns the metadata for a single package +func PackageMetadata(ctx *context.Context) { + packageName := normalizer.Replace(ctx.Params("id")) + + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypePyPI, packageName) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pvs) == 0 { + apiError(ctx, http.StatusNotFound, err) + return + } + + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.Data["RegistryURL"] = setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/pypi" + ctx.Data["PackageDescriptor"] = pds[0] + ctx.Data["PackageDescriptors"] = pds + ctx.Render = templates.HTMLRenderer() + ctx.HTML(http.StatusOK, "api/packages/pypi/simple") +} + +// DownloadPackageFile serves the content of a package +func DownloadPackageFile(ctx *context.Context) { + packageName := normalizer.Replace(ctx.Params("id")) + packageVersion := ctx.Params("version") + filename := ctx.Params("filename") + + s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + ctx, + &packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypePyPI, + Name: packageName, + Version: packageVersion, + }, + &packages_service.PackageFileInfo{ + Filename: filename, + }, + ) + if err != nil { + if err == packages_model.ErrPackageNotExist || err == packages_model.ErrPackageFileNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer s.Close() + + ctx.ServeStream(s, pf.Name) +} + +// UploadPackageFile adds a file to the package. If the package does not exist, it gets created. +func UploadPackageFile(ctx *context.Context) { + file, fileHeader, err := ctx.Req.FormFile("content") + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + defer file.Close() + + buf, err := packages_module.CreateHashedBufferFromReader(file, 32*1024*1024) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + _, _, hashSHA256, _ := buf.Sums() + + if !strings.EqualFold(ctx.Req.FormValue("sha256_digest"), fmt.Sprintf("%x", hashSHA256)) { + apiError(ctx, http.StatusBadRequest, "hash mismatch") + return + } + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + packageName := normalizer.Replace(ctx.Req.FormValue("name")) + packageVersion := ctx.Req.FormValue("version") + if !nameMatcher.MatchString(packageName) || !versionMatcher.MatchString(packageVersion) { + apiError(ctx, http.StatusBadRequest, "invalid name or version") + return + } + + projectURL := ctx.Req.FormValue("home_page") + if !validation.IsValidURL(projectURL) { + projectURL = "" + } + + _, _, err = packages_service.CreatePackageOrAddFileToExisting( + &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypePyPI, + Name: packageName, + Version: packageVersion, + }, + SemverCompatible: true, + Creator: ctx.Doer, + Metadata: &pypi_module.Metadata{ + Author: ctx.Req.FormValue("author"), + Description: ctx.Req.FormValue("description"), + LongDescription: ctx.Req.FormValue("long_description"), + Summary: ctx.Req.FormValue("summary"), + ProjectURL: projectURL, + License: ctx.Req.FormValue("license"), + RequiresPython: ctx.Req.FormValue("requires_python"), + }, + }, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: fileHeader.Filename, + }, + Data: buf, + IsLead: true, + }, + ) + if err != nil { + if err == packages_model.ErrDuplicatePackageFile { + apiError(ctx, http.StatusBadRequest, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.Status(http.StatusCreated) +} diff --git a/routers/api/packages/rubygems/rubygems.go b/routers/api/packages/rubygems/rubygems.go new file mode 100644 index 0000000000..a5a9b779ab --- /dev/null +++ b/routers/api/packages/rubygems/rubygems.go @@ -0,0 +1,285 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package rubygems + +import ( + "compress/gzip" + "compress/zlib" + "fmt" + "io" + "net/http" + "strings" + + packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/context" + packages_module "code.gitea.io/gitea/modules/packages" + rubygems_module "code.gitea.io/gitea/modules/packages/rubygems" + "code.gitea.io/gitea/routers/api/packages/helper" + packages_service "code.gitea.io/gitea/services/packages" +) + +func apiError(ctx *context.Context, status int, obj interface{}) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { + ctx.PlainText(status, message) + }) +} + +// EnumeratePackages serves the package list +func EnumeratePackages(ctx *context.Context) { + packages, err := packages_model.GetVersionsByPackageType(ctx, ctx.Package.Owner.ID, packages_model.TypeRubyGems) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + enumeratePackages(ctx, "specs.4.8", packages) +} + +// EnumeratePackagesLatest serves the list of the lastest version of every package +func EnumeratePackagesLatest(ctx *context.Context) { + pvs, _, err := packages_model.SearchLatestVersions(ctx, &packages_model.PackageSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + Type: string(packages_model.TypeRubyGems), + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + enumeratePackages(ctx, "latest_specs.4.8", pvs) +} + +// EnumeratePackagesPreRelease is not supported and serves an empty list +func EnumeratePackagesPreRelease(ctx *context.Context) { + enumeratePackages(ctx, "prerelease_specs.4.8", []*packages_model.PackageVersion{}) +} + +func enumeratePackages(ctx *context.Context, filename string, pvs []*packages_model.PackageVersion) { + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + specs := make([]interface{}, 0, len(pds)) + for _, p := range pds { + specs = append(specs, []interface{}{ + p.Package.Name, + &rubygems_module.RubyUserMarshal{ + Name: "Gem::Version", + Value: []string{p.Version.Version}, + }, + p.Metadata.(*rubygems_module.Metadata).Platform, + }) + } + + ctx.SetServeHeaders(filename + ".gz") + + zw := gzip.NewWriter(ctx.Resp) + defer zw.Close() + + zw.Name = filename + + if err := rubygems_module.NewMarshalEncoder(zw).Encode(specs); err != nil { + ctx.ServerError("Download file failed", err) + } +} + +// ServePackageSpecification serves the compressed Gemspec file of a package +func ServePackageSpecification(ctx *context.Context) { + filename := ctx.Params("filename") + + if !strings.HasSuffix(filename, ".gemspec.rz") { + apiError(ctx, http.StatusNotImplemented, nil) + return + } + + pvs, err := packages_model.GetVersionsByFilename(ctx, ctx.Package.Owner.ID, packages_model.TypeRubyGems, filename[:len(filename)-10]+"gem") + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if len(pvs) != 1 { + apiError(ctx, http.StatusNotFound, nil) + return + } + + pd, err := packages_model.GetPackageDescriptor(ctx, pvs[0]) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.SetServeHeaders(filename) + + zw := zlib.NewWriter(ctx.Resp) + defer zw.Close() + + metadata := pd.Metadata.(*rubygems_module.Metadata) + + // create a Ruby Gem::Specification object + spec := &rubygems_module.RubyUserDef{ + Name: "Gem::Specification", + Value: []interface{}{ + "3.2.3", // @rubygems_version + 4, // @specification_version, + pd.Package.Name, + &rubygems_module.RubyUserMarshal{ + Name: "Gem::Version", + Value: []string{pd.Version.Version}, + }, + nil, // date + metadata.Summary, // @summary + nil, // @required_ruby_version + nil, // @required_rubygems_version + metadata.Platform, // @original_platform + []interface{}{}, // @dependencies + nil, // rubyforge_project + "", // @email + metadata.Authors, + metadata.Description, + metadata.ProjectURL, + true, // has_rdoc + metadata.Platform, // @new_platform + nil, + metadata.Licenses, + }, + } + + if err := rubygems_module.NewMarshalEncoder(zw).Encode(spec); err != nil { + ctx.ServerError("Download file failed", err) + } +} + +// DownloadPackageFile serves the content of a package +func DownloadPackageFile(ctx *context.Context) { + filename := ctx.Params("filename") + + pvs, err := packages_model.GetVersionsByFilename(ctx, ctx.Package.Owner.ID, packages_model.TypeRubyGems, filename) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if len(pvs) != 1 { + apiError(ctx, http.StatusNotFound, nil) + return + } + + s, pf, err := packages_service.GetFileStreamByPackageVersion( + ctx, + pvs[0], + &packages_service.PackageFileInfo{ + Filename: filename, + }, + ) + if err != nil { + if err == packages_model.ErrPackageFileNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer s.Close() + + ctx.ServeStream(s, pf.Name) +} + +// UploadPackageFile adds a file to the package. If the package does not exist, it gets created. +func UploadPackageFile(ctx *context.Context) { + upload, close, err := ctx.UploadStream() + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + if close { + defer upload.Close() + } + + buf, err := packages_module.CreateHashedBufferFromReader(upload, 32*1024*1024) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + rp, err := rubygems_module.ParsePackageMetaData(buf) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if _, err := buf.Seek(0, io.SeekStart); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + var filename string + if rp.Metadata.Platform == "" || rp.Metadata.Platform == "ruby" { + filename = strings.ToLower(fmt.Sprintf("%s-%s.gem", rp.Name, rp.Version)) + } else { + filename = strings.ToLower(fmt.Sprintf("%s-%s-%s.gem", rp.Name, rp.Version, rp.Metadata.Platform)) + } + + _, _, err = packages_service.CreatePackageAndAddFile( + &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeRubyGems, + Name: rp.Name, + Version: rp.Version, + }, + SemverCompatible: true, + Creator: ctx.Doer, + Metadata: rp.Metadata, + }, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: filename, + }, + Data: buf, + IsLead: true, + }, + ) + if err != nil { + if err == packages_model.ErrDuplicatePackageVersion { + apiError(ctx, http.StatusBadRequest, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.Status(http.StatusCreated) +} + +// DeletePackage deletes a package +func DeletePackage(ctx *context.Context) { + // Go populates the form only for POST, PUT and PATCH requests + if err := ctx.Req.ParseMultipartForm(32 << 20); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + packageName := ctx.FormString("gem_name") + packageVersion := ctx.FormString("version") + + err := packages_service.RemovePackageVersionByNameAndVersion( + ctx.Doer, + &packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeRubyGems, + Name: packageName, + Version: packageVersion, + }, + ) + if err != nil { + if err == packages_model.ErrPackageNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + } +} |