From 1d332342db6d5bd4e1552d8d46720bf1b948c26b Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 30 Mar 2022 10:42:47 +0200 Subject: Add Package Registry (#16510) * Added package store settings. * Added models. * Added generic package registry. * Added tests. * Added NuGet package registry. * Moved service index to api file. * Added NPM package registry. * Added Maven package registry. * Added PyPI package registry. * Summary is deprecated. * Changed npm name. * Sanitize project url. * Allow only scoped packages. * Added user interface. * Changed method name. * Added missing migration file. * Set page info. * Added documentation. * Added documentation links. * Fixed wrong error message. * Lint template files. * Fixed merge errors. * Fixed unit test storage path. * Switch to json module. * Added suggestions. * Added package webhook. * Add package api. * Fixed swagger file. * Fixed enum and comments. * Fixed NuGet pagination. * Print test names. * Added api tests. * Fixed access level. * Fix User unmarshal. * Added RubyGems package registry. * Fix lint. * Implemented io.Writer. * Added support for sha256/sha512 checksum files. * Improved maven-metadata.xml support. * Added support for symbol package uploads. * Added tests. * Added overview docs. * Added npm dependencies and keywords. * Added no-packages information. * Display file size. * Display asset count. * Fixed filter alignment. * Added package icons. * Formatted instructions. * Allow anonymous package downloads. * Fixed comments. * Fixed postgres test. * Moved file. * Moved models to models/packages. * Use correct error response format per client. * Use simpler search form. * Fixed IsProd. * Restructured data model. * Prevent empty filename. * Fix swagger. * Implemented user/org registry. * Implemented UI. * Use GetUserByIDCtx. * Use table for dependencies. * make svg * Added support for unscoped npm packages. * Add support for npm dist tags. * Added tests for npm tags. * Unlink packages if repository gets deleted. * Prevent user/org delete if a packages exist. * Use package unlink in repository service. * Added support for composer packages. * Restructured package docs. * Added missing tests. * Fixed generic content page. * Fixed docs. * Fixed swagger. * Added missing type. * Fixed ambiguous column. * Organize content store by sha256 hash. * Added admin package management. * Added support for sorting. * Add support for multiple identical versions/files. * Added missing repository unlink. * Added file properties. * make fmt * lint * Added Conan package registry. * Updated docs. * Unify package names. * Added swagger enum. * Use longer TEXT column type. * Removed version composite key. * Merged package and container registry. * Removed index. * Use dedicated package router. * Moved files to new location. * Updated docs. * Fixed JOIN order. * Fixed GROUP BY statement. * Fixed GROUP BY #2. * Added symbol server support. * Added more tests. * Set NOT NULL. * Added setting to disable package registries. * Moved auth into service. * refactor * Use ctx everywhere. * Added package cleanup task. * Changed packages path. * Added container registry. * Refactoring * Updated comparison. * Fix swagger. * Fixed table order. * Use token auth for npm routes. * Enabled ReverseProxy auth. * Added packages link for orgs. * Fixed anonymous org access. * Enable copy button for setup instructions. * Merge error * Added suggestions. * Fixed merge. * Handle "generic". * Added link for TODO. * Added suggestions. * Changed temporary buffer filename. * Added suggestions. * Apply suggestions from code review Co-authored-by: Thomas Boerger * Update docs/content/doc/packages/nuget.en-us.md Co-authored-by: wxiaoguang Co-authored-by: Thomas Boerger --- models/packages/container/const.go | 10 ++ models/packages/container/search.go | 227 ++++++++++++++++++++++++++++++++++++ 2 files changed, 237 insertions(+) create mode 100644 models/packages/container/const.go create mode 100644 models/packages/container/search.go (limited to 'models/packages/container') diff --git a/models/packages/container/const.go b/models/packages/container/const.go new file mode 100644 index 0000000000..9d3ed64a6e --- /dev/null +++ b/models/packages/container/const.go @@ -0,0 +1,10 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package container + +const ( + ManifestFilename = "manifest.json" + UploadVersion = "_upload" +) diff --git a/models/packages/container/search.go b/models/packages/container/search.go new file mode 100644 index 0000000000..972cac9528 --- /dev/null +++ b/models/packages/container/search.go @@ -0,0 +1,227 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package container + +import ( + "context" + "errors" + "strings" + "time" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" + container_module "code.gitea.io/gitea/modules/packages/container" + + "xorm.io/builder" +) + +var ErrContainerBlobNotExist = errors.New("Container blob does not exist") + +type BlobSearchOptions struct { + OwnerID int64 + Image string + Digest string + Tag string + IsManifest bool +} + +func (opts *BlobSearchOptions) toConds() builder.Cond { + var cond builder.Cond = builder.Eq{ + "package.type": packages.TypeContainer, + } + + if opts.OwnerID != 0 { + cond = cond.And(builder.Eq{"package.owner_id": opts.OwnerID}) + } + if opts.Image != "" { + cond = cond.And(builder.Eq{"package.lower_name": strings.ToLower(opts.Image)}) + } + if opts.Tag != "" { + cond = cond.And(builder.Eq{"package_version.lower_version": strings.ToLower(opts.Tag)}) + } + if opts.IsManifest { + cond = cond.And(builder.Eq{"package_file.lower_name": ManifestFilename}) + } + if opts.Digest != "" { + var propsCond builder.Cond = builder.Eq{ + "package_property.ref_type": packages.PropertyTypeFile, + "package_property.name": container_module.PropertyDigest, + "package_property.value": opts.Digest, + } + + cond = cond.And(builder.In("package_file.id", builder.Select("package_property.ref_id").Where(propsCond).From("package_property"))) + } + + return cond +} + +// GetContainerBlob gets the container blob matching the blob search options +// If multiple matching blobs are found (manifests with the same digest) the first (according to the database) is selected. +func GetContainerBlob(ctx context.Context, opts *BlobSearchOptions) (*packages.PackageFileDescriptor, error) { + pfds, err := getContainerBlobsLimit(ctx, opts, 1) + if err != nil { + return nil, err + } + if len(pfds) != 1 { + return nil, ErrContainerBlobNotExist + } + + return pfds[0], nil +} + +// GetContainerBlobs gets the container blobs matching the blob search options +func GetContainerBlobs(ctx context.Context, opts *BlobSearchOptions) ([]*packages.PackageFileDescriptor, error) { + return getContainerBlobsLimit(ctx, opts, 0) +} + +func getContainerBlobsLimit(ctx context.Context, opts *BlobSearchOptions, limit int) ([]*packages.PackageFileDescriptor, error) { + pfs := make([]*packages.PackageFile, 0, limit) + sess := db.GetEngine(ctx). + Join("INNER", "package_version", "package_version.id = package_file.version_id"). + Join("INNER", "package", "package.id = package_version.package_id"). + Where(opts.toConds()) + + if limit > 0 { + sess = sess.Limit(limit) + } + + if err := sess.Find(&pfs); err != nil { + return nil, err + } + + pfds := make([]*packages.PackageFileDescriptor, 0, len(pfs)) + for _, pf := range pfs { + pfd, err := packages.GetPackageFileDescriptor(ctx, pf) + if err != nil { + return nil, err + } + pfds = append(pfds, pfd) + } + + return pfds, nil +} + +// GetManifestVersions gets all package versions representing the matching manifest +func GetManifestVersions(ctx context.Context, opts *BlobSearchOptions) ([]*packages.PackageVersion, error) { + cond := opts.toConds().And(builder.Eq{"package_version.is_internal": false}) + + pvs := make([]*packages.PackageVersion, 0, 10) + return pvs, db.GetEngine(ctx). + Join("INNER", "package", "package.id = package_version.package_id"). + Join("INNER", "package_file", "package_file.version_id = package_version.id"). + Where(cond). + Find(&pvs) +} + +// GetImageTags gets a sorted list of the tags of an image +// The result is suitable for the api call. +func GetImageTags(ctx context.Context, ownerID int64, image string, n int, last string) ([]string, error) { + // Short circuit: n == 0 should return an empty list + if n == 0 { + return []string{}, nil + } + + var cond builder.Cond = builder.Eq{ + "package.type": packages.TypeContainer, + "package.owner_id": ownerID, + "package.lower_name": strings.ToLower(image), + "package_version.is_internal": false, + } + + var propsCond builder.Cond = builder.Eq{ + "package_property.ref_type": packages.PropertyTypeVersion, + "package_property.name": container_module.PropertyManifestTagged, + } + + cond = cond.And(builder.In("package_version.id", builder.Select("package_property.ref_id").Where(propsCond).From("package_property"))) + + if last != "" { + cond = cond.And(builder.Gt{"package_version.lower_version": strings.ToLower(last)}) + } + + sess := db.GetEngine(ctx). + Table("package_version"). + Select("package_version.lower_version"). + Join("INNER", "package", "package.id = package_version.package_id"). + Where(cond). + Asc("package_version.lower_version") + + var tags []string + if n > 0 { + sess = sess.Limit(n) + + tags = make([]string, 0, n) + } else { + tags = make([]string, 0, 10) + } + + return tags, sess.Find(&tags) +} + +type ImageTagsSearchOptions struct { + PackageID int64 + Query string + IsTagged bool + db.Paginator +} + +func (opts *ImageTagsSearchOptions) toConds() builder.Cond { + var cond builder.Cond = builder.Eq{ + "package.type": packages.TypeContainer, + "package.id": opts.PackageID, + "package_version.is_internal": false, + } + + if opts.Query != "" { + cond = cond.And(builder.Like{"package_version.lower_version", strings.ToLower(opts.Query)}) + } + + var propsCond builder.Cond = builder.Eq{ + "package_property.ref_type": packages.PropertyTypeVersion, + "package_property.name": container_module.PropertyManifestTagged, + } + + in := builder.In("package_version.id", builder.Select("package_property.ref_id").Where(propsCond).From("package_property")) + + if opts.IsTagged { + cond = cond.And(in) + } else { + cond = cond.And(builder.Not{in}) + } + + return cond +} + +// SearchImageTags gets a sorted list of the tags of an image +func SearchImageTags(ctx context.Context, opts *ImageTagsSearchOptions) ([]*packages.PackageVersion, int64, error) { + sess := db.GetEngine(ctx). + Join("INNER", "package", "package.id = package_version.package_id"). + Where(opts.toConds()). + Desc("package_version.created_unix") + + if opts.Paginator != nil { + sess = db.SetSessionPagination(sess, opts) + } + + pvs := make([]*packages.PackageVersion, 0, 10) + count, err := sess.FindAndCount(&pvs) + return pvs, count, err +} + +func SearchExpiredUploadedBlobs(ctx context.Context, olderThan time.Duration) ([]*packages.PackageFile, error) { + var cond builder.Cond = builder.Eq{ + "package_version.is_internal": true, + "package_version.lower_version": UploadVersion, + "package.type": packages.TypeContainer, + } + cond = cond.And(builder.Lt{"package_file.created_unix": time.Now().Add(-olderThan).Unix()}) + + var pfs []*packages.PackageFile + return pfs, db.GetEngine(ctx). + Join("INNER", "package_version", "package_version.id = package_file.version_id"). + Join("INNER", "package", "package.id = package_version.package_id"). + Where(cond). + Find(&pfs) +} -- cgit v1.2.3