Add Package Registry (#16510)

* Added package store settings.

* Added models.

* Added generic package registry.

* Added tests.

* Added NuGet package registry.

* Moved service index to api file.

* Added NPM package registry.

* Added Maven package registry.

* Added PyPI package registry.

* Summary is deprecated.

* Changed npm name.

* Sanitize project url.

* Allow only scoped packages.

* Added user interface.

* Changed method name.

* Added missing migration file.

* Set page info.

* Added documentation.

* Added documentation links.

* Fixed wrong error message.

* Lint template files.

* Fixed merge errors.

* Fixed unit test storage path.

* Switch to json module.

* Added suggestions.

* Added package webhook.

* Add package api.

* Fixed swagger file.

* Fixed enum and comments.

* Fixed NuGet pagination.

* Print test names.

* Added api tests.

* Fixed access level.

* Fix User unmarshal.

* Added RubyGems package registry.

* Fix lint.

* Implemented io.Writer.

* Added support for sha256/sha512 checksum files.

* Improved maven-metadata.xml support.

* Added support for symbol package uploads.

* Added tests.

* Added overview docs.

* Added npm dependencies and keywords.

* Added no-packages information.

* Display file size.

* Display asset count.

* Fixed filter alignment.

* Added package icons.

* Formatted instructions.

* Allow anonymous package downloads.

* Fixed comments.

* Fixed postgres test.

* Moved file.

* Moved models to models/packages.

* Use correct error response format per client.

* Use simpler search form.

* Fixed IsProd.

* Restructured data model.

* Prevent empty filename.

* Fix swagger.

* Implemented user/org registry.

* Implemented UI.

* Use GetUserByIDCtx.

* Use table for dependencies.

* make svg

* Added support for unscoped npm packages.

* Add support for npm dist tags.

* Added tests for npm tags.

* Unlink packages if repository gets deleted.

* Prevent user/org delete if a packages exist.

* Use package unlink in repository service.

* Added support for composer packages.

* Restructured package docs.

* Added missing tests.

* Fixed generic content page.

* Fixed docs.

* Fixed swagger.

* Added missing type.

* Fixed ambiguous column.

* Organize content store by sha256 hash.

* Added admin package management.

* Added support for sorting.

* Add support for multiple identical versions/files.

* Added missing repository unlink.

* Added file properties.

* make fmt

* lint

* Added Conan package registry.

* Updated docs.

* Unify package names.

* Added swagger enum.

* Use longer TEXT column type.

* Removed version composite key.

* Merged package and container registry.

* Removed index.

* Use dedicated package router.

* Moved files to new location.

* Updated docs.

* Fixed JOIN order.

* Fixed GROUP BY statement.

* Fixed GROUP BY #2.

* Added symbol server support.

* Added more tests.

* Set NOT NULL.

* Added setting to disable package registries.

* Moved auth into service.

* refactor

* Use ctx everywhere.

* Added package cleanup task.

* Changed packages path.

* Added container registry.

* Refactoring

* Updated comparison.

* Fix swagger.

* Fixed table order.

* Use token auth for npm routes.

* Enabled ReverseProxy auth.

* Added packages link for orgs.

* Fixed anonymous org access.

* Enable copy button for setup instructions.

* Merge error

* Added suggestions.

* Fixed merge.

* Handle "generic".

* Added link for TODO.

* Added suggestions.

* Changed temporary buffer filename.

* Added suggestions.

* Apply suggestions from code review

Co-authored-by: Thomas Boerger <thomas@webhippie.de>

* Update docs/content/doc/packages/nuget.en-us.md

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: Thomas Boerger <thomas@webhippie.de>

1 files changed, 192 insertions, 0 deletions

diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go
new file mode 100644
index 0000000000..3249260f80
--- /dev/null
+++ b/models/packages/descriptor.go
@@ -0,0 +1,192 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package packages
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+
+	repo_model "code.gitea.io/gitea/models/repo"
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/json"
+	"code.gitea.io/gitea/modules/packages/composer"
+	"code.gitea.io/gitea/modules/packages/conan"
+	"code.gitea.io/gitea/modules/packages/container"
+	"code.gitea.io/gitea/modules/packages/maven"
+	"code.gitea.io/gitea/modules/packages/npm"
+	"code.gitea.io/gitea/modules/packages/nuget"
+	"code.gitea.io/gitea/modules/packages/pypi"
+	"code.gitea.io/gitea/modules/packages/rubygems"
+
+	"github.com/hashicorp/go-version"
+)
+
+// PackagePropertyList is a list of package properties
+type PackagePropertyList []*PackageProperty
+
+// GetByName gets the first property value with the specific name
+func (l PackagePropertyList) GetByName(name string) string {
+	for _, pp := range l {
+		if pp.Name == name {
+			return pp.Value
+		}
+	}
+	return ""
+}
+
+// PackageDescriptor describes a package
+type PackageDescriptor struct {
+	Package    *Package
+	Owner      *user_model.User
+	Repository *repo_model.Repository
+	Version    *PackageVersion
+	SemVer     *version.Version
+	Creator    *user_model.User
+	Properties PackagePropertyList
+	Metadata   interface{}
+	Files      []*PackageFileDescriptor
+}
+
+// PackageFileDescriptor describes a package file
+type PackageFileDescriptor struct {
+	File       *PackageFile
+	Blob       *PackageBlob
+	Properties PackagePropertyList
+}
+
+// PackageWebLink returns the package web link
+func (pd *PackageDescriptor) PackageWebLink() string {
+	return fmt.Sprintf("%s/-/packages/%s/%s", pd.Owner.HTMLURL(), string(pd.Package.Type), url.PathEscape(pd.Package.LowerName))
+}
+
+// FullWebLink returns the package version web link
+func (pd *PackageDescriptor) FullWebLink() string {
+	return fmt.Sprintf("%s/%s", pd.PackageWebLink(), url.PathEscape(pd.Version.LowerVersion))
+}
+
+// CalculateBlobSize returns the total blobs size in bytes
+func (pd *PackageDescriptor) CalculateBlobSize() int64 {
+	size := int64(0)
+	for _, f := range pd.Files {
+		size += f.Blob.Size
+	}
+	return size
+}
+
+// GetPackageDescriptor gets the package description for a version
+func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDescriptor, error) {
+	p, err := GetPackageByID(ctx, pv.PackageID)
+	if err != nil {
+		return nil, err
+	}
+	o, err := user_model.GetUserByIDCtx(ctx, p.OwnerID)
+	if err != nil {
+		return nil, err
+	}
+	repository, err := repo_model.GetRepositoryByIDCtx(ctx, p.RepoID)
+	if err != nil && !repo_model.IsErrRepoNotExist(err) {
+		return nil, err
+	}
+	creator, err := user_model.GetUserByIDCtx(ctx, pv.CreatorID)
+	if err != nil {
+		return nil, err
+	}
+	var semVer *version.Version
+	if p.SemverCompatible {
+		semVer, err = version.NewVersion(pv.Version)
+		if err != nil {
+			return nil, err
+		}
+	}
+	pvps, err := GetProperties(ctx, PropertyTypeVersion, pv.ID)
+	if err != nil {
+		return nil, err
+	}
+	pfs, err := GetFilesByVersionID(ctx, pv.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	pfds := make([]*PackageFileDescriptor, 0, len(pfs))
+	for _, pf := range pfs {
+		pfd, err := GetPackageFileDescriptor(ctx, pf)
+		if err != nil {
+			return nil, err
+		}
+		pfds = append(pfds, pfd)
+	}
+
+	var metadata interface{}
+	switch p.Type {
+	case TypeComposer:
+		metadata = &composer.Metadata{}
+	case TypeConan:
+		metadata = &conan.Metadata{}
+	case TypeContainer:
+		metadata = &container.Metadata{}
+	case TypeGeneric:
+		// generic packages have no metadata
+	case TypeNuGet:
+		metadata = &nuget.Metadata{}
+	case TypeNpm:
+		metadata = &npm.Metadata{}
+	case TypeMaven:
+		metadata = &maven.Metadata{}
+	case TypePyPI:
+		metadata = &pypi.Metadata{}
+	case TypeRubyGems:
+		metadata = &rubygems.Metadata{}
+	default:
+		panic(fmt.Sprintf("unknown package type: %s", string(p.Type)))
+	}
+	if metadata != nil {
+		if err := json.Unmarshal([]byte(pv.MetadataJSON), &metadata); err != nil {
+			return nil, err
+		}
+	}
+
+	return &PackageDescriptor{
+		Package:    p,
+		Owner:      o,
+		Repository: repository,
+		Version:    pv,
+		SemVer:     semVer,
+		Creator:    creator,
+		Properties: PackagePropertyList(pvps),
+		Metadata:   metadata,
+		Files:      pfds,
+	}, nil
+}
+
+// GetPackageFileDescriptor gets a package file descriptor for a package file
+func GetPackageFileDescriptor(ctx context.Context, pf *PackageFile) (*PackageFileDescriptor, error) {
+	pb, err := GetBlobByID(ctx, pf.BlobID)
+	if err != nil {
+		return nil, err
+	}
+	pfps, err := GetProperties(ctx, PropertyTypeFile, pf.ID)
+	if err != nil {
+		return nil, err
+	}
+	return &PackageFileDescriptor{
+		pf,
+		pb,
+		PackagePropertyList(pfps),
+	}, nil
+}
+
+// GetPackageDescriptors gets the package descriptions for the versions
+func GetPackageDescriptors(ctx context.Context, pvs []*PackageVersion) ([]*PackageDescriptor, error) {
+	pds := make([]*PackageDescriptor, 0, len(pvs))
+	for _, pv := range pvs {
+		pd, err := GetPackageDescriptor(ctx, pv)
+		if err != nil {
+			return nil, err
+		}
+		pds = append(pds, pd)
+	}
+	return pds, nil
+}