Add Package Registry (#16510)

* Added package store settings.

* Added models.

* Added generic package registry.

* Added tests.

* Added NuGet package registry.

* Moved service index to api file.

* Added NPM package registry.

* Added Maven package registry.

* Added PyPI package registry.

* Summary is deprecated.

* Changed npm name.

* Sanitize project url.

* Allow only scoped packages.

* Added user interface.

* Changed method name.

* Added missing migration file.

* Set page info.

* Added documentation.

* Added documentation links.

* Fixed wrong error message.

* Lint template files.

* Fixed merge errors.

* Fixed unit test storage path.

* Switch to json module.

* Added suggestions.

* Added package webhook.

* Add package api.

* Fixed swagger file.

* Fixed enum and comments.

* Fixed NuGet pagination.

* Print test names.

* Added api tests.

* Fixed access level.

* Fix User unmarshal.

* Added RubyGems package registry.

* Fix lint.

* Implemented io.Writer.

* Added support for sha256/sha512 checksum files.

* Improved maven-metadata.xml support.

* Added support for symbol package uploads.

* Added tests.

* Added overview docs.

* Added npm dependencies and keywords.

* Added no-packages information.

* Display file size.

* Display asset count.

* Fixed filter alignment.

* Added package icons.

* Formatted instructions.

* Allow anonymous package downloads.

* Fixed comments.

* Fixed postgres test.

* Moved file.

* Moved models to models/packages.

* Use correct error response format per client.

* Use simpler search form.

* Fixed IsProd.

* Restructured data model.

* Prevent empty filename.

* Fix swagger.

* Implemented user/org registry.

* Implemented UI.

* Use GetUserByIDCtx.

* Use table for dependencies.

* make svg

* Added support for unscoped npm packages.

* Add support for npm dist tags.

* Added tests for npm tags.

* Unlink packages if repository gets deleted.

* Prevent user/org delete if a packages exist.

* Use package unlink in repository service.

* Added support for composer packages.

* Restructured package docs.

* Added missing tests.

* Fixed generic content page.

* Fixed docs.

* Fixed swagger.

* Added missing type.

* Fixed ambiguous column.

* Organize content store by sha256 hash.

* Added admin package management.

* Added support for sorting.

* Add support for multiple identical versions/files.

* Added missing repository unlink.

* Added file properties.

* make fmt

* lint

* Added Conan package registry.

* Updated docs.

* Unify package names.

* Added swagger enum.

* Use longer TEXT column type.

* Removed version composite key.

* Merged package and container registry.

* Removed index.

* Use dedicated package router.

* Moved files to new location.

* Updated docs.

* Fixed JOIN order.

* Fixed GROUP BY statement.

* Fixed GROUP BY #2.

* Added symbol server support.

* Added more tests.

* Set NOT NULL.

* Added setting to disable package registries.

* Moved auth into service.

* refactor

* Use ctx everywhere.

* Added package cleanup task.

* Changed packages path.

* Added container registry.

* Refactoring

* Updated comparison.

* Fix swagger.

* Fixed table order.

* Use token auth for npm routes.

* Enabled ReverseProxy auth.

* Added packages link for orgs.

* Fixed anonymous org access.

* Enable copy button for setup instructions.

* Merge error

* Added suggestions.

* Fixed merge.

* Handle "generic".

* Added link for TODO.

* Added suggestions.

* Changed temporary buffer filename.

* Added suggestions.

* Apply suggestions from code review

Co-authored-by: Thomas Boerger <thomas@webhippie.de>

* Update docs/content/doc/packages/nuget.en-us.md

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: Thomas Boerger <thomas@webhippie.de>

2 files changed, 320 insertions, 0 deletions

diff --git a/models/packages/conan/references.go b/models/packages/conan/references.go
new file mode 100644
index 0000000000..4b7b201430
--- /dev/null
+++ b/models/packages/conan/references.go
@@ -0,0 +1,171 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package conan
+
+import (
+	"context"
+	"errors"
+	"strconv"
+	"strings"
+
+	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/models/packages"
+	conan_module "code.gitea.io/gitea/modules/packages/conan"
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"xorm.io/builder"
+)
+
+var (
+	ErrRecipeReferenceNotExist  = errors.New("Recipe reference does not exist")
+	ErrPackageReferenceNotExist = errors.New("Package reference does not exist")
+)
+
+// RecipeExists checks if a recipe exists
+func RecipeExists(ctx context.Context, ownerID int64, ref *conan_module.RecipeReference) (bool, error) {
+	revisions, err := GetRecipeRevisions(ctx, ownerID, ref)
+	if err != nil {
+		return false, err
+	}
+
+	return len(revisions) != 0, nil
+}
+
+type PropertyValue struct {
+	Value       string
+	CreatedUnix timeutil.TimeStamp
+}
+
+func findPropertyValues(ctx context.Context, propertyName string, ownerID int64, name, version string, propertyFilter map[string]string) ([]*PropertyValue, error) {
+	var propsCond builder.Cond = builder.Eq{
+		"package_property.ref_type": packages.PropertyTypeFile,
+	}
+	propsCond = propsCond.And(builder.Expr("package_property.ref_id = package_file.id"))
+
+	propsCondBlock := builder.NewCond()
+	for name, value := range propertyFilter {
+		propsCondBlock = propsCondBlock.Or(builder.Eq{
+			"package_property.name":  name,
+			"package_property.value": value,
+		})
+	}
+	propsCond = propsCond.And(propsCondBlock)
+
+	var cond builder.Cond = builder.Eq{
+		"package.type":                    packages.TypeConan,
+		"package.owner_id":                ownerID,
+		"package.lower_name":              strings.ToLower(name),
+		"package_version.lower_version":   strings.ToLower(version),
+		"package_version.is_internal":     false,
+		strconv.Itoa(len(propertyFilter)): builder.Select("COUNT(*)").Where(propsCond).From("package_property"),
+	}
+
+	in2 := builder.
+		Select("package_file.id").
+		From("package_file").
+		Join("INNER", "package_version", "package_version.id = package_file.version_id").
+		Join("INNER", "package", "package.id = package_version.package_id").
+		Where(cond)
+
+	query := builder.
+		Select("package_property.value, MAX(package_file.created_unix) AS created_unix").
+		From("package_property").
+		Join("INNER", "package_file", "package_file.id = package_property.ref_id").
+		Where(builder.Eq{"package_property.name": propertyName}.And(builder.In("package_property.ref_id", in2))).
+		GroupBy("package_property.value").
+		OrderBy("created_unix DESC")
+
+	var values []*PropertyValue
+	return values, db.GetEngine(ctx).SQL(query).Find(&values)
+}
+
+// GetRecipeRevisions gets all revisions of a recipe
+func GetRecipeRevisions(ctx context.Context, ownerID int64, ref *conan_module.RecipeReference) ([]*PropertyValue, error) {
+	values, err := findPropertyValues(
+		ctx,
+		conan_module.PropertyRecipeRevision,
+		ownerID,
+		ref.Name,
+		ref.Version,
+		map[string]string{
+			conan_module.PropertyRecipeUser:    ref.User,
+			conan_module.PropertyRecipeChannel: ref.Channel,
+		},
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return values, nil
+}
+
+// GetLastRecipeRevision gets the latest recipe revision
+func GetLastRecipeRevision(ctx context.Context, ownerID int64, ref *conan_module.RecipeReference) (*PropertyValue, error) {
+	revisions, err := GetRecipeRevisions(ctx, ownerID, ref)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(revisions) == 0 {
+		return nil, ErrRecipeReferenceNotExist
+	}
+	return revisions[0], nil
+}
+
+// GetPackageReferences gets all package references of a recipe
+func GetPackageReferences(ctx context.Context, ownerID int64, ref *conan_module.RecipeReference) ([]*PropertyValue, error) {
+	values, err := findPropertyValues(
+		ctx,
+		conan_module.PropertyPackageReference,
+		ownerID,
+		ref.Name,
+		ref.Version,
+		map[string]string{
+			conan_module.PropertyRecipeUser:     ref.User,
+			conan_module.PropertyRecipeChannel:  ref.Channel,
+			conan_module.PropertyRecipeRevision: ref.Revision,
+		},
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return values, nil
+}
+
+// GetPackageRevisions gets all revision of a package
+func GetPackageRevisions(ctx context.Context, ownerID int64, ref *conan_module.PackageReference) ([]*PropertyValue, error) {
+	values, err := findPropertyValues(
+		ctx,
+		conan_module.PropertyPackageRevision,
+		ownerID,
+		ref.Recipe.Name,
+		ref.Recipe.Version,
+		map[string]string{
+			conan_module.PropertyRecipeUser:       ref.Recipe.User,
+			conan_module.PropertyRecipeChannel:    ref.Recipe.Channel,
+			conan_module.PropertyRecipeRevision:   ref.Recipe.Revision,
+			conan_module.PropertyPackageReference: ref.Reference,
+		},
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return values, nil
+}
+
+// GetLastPackageRevision gets the latest package revision
+func GetLastPackageRevision(ctx context.Context, ownerID int64, ref *conan_module.PackageReference) (*PropertyValue, error) {
+	revisions, err := GetPackageRevisions(ctx, ownerID, ref)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(revisions) == 0 {
+		return nil, ErrPackageReferenceNotExist
+	}
+	return revisions[0], nil
+}
diff --git a/models/packages/conan/search.go b/models/packages/conan/search.go
new file mode 100644
index 0000000000..c274a7ce02
--- /dev/null
+++ b/models/packages/conan/search.go
@@ -0,0 +1,149 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package conan
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/models/packages"
+	conan_module "code.gitea.io/gitea/modules/packages/conan"
+
+	"xorm.io/builder"
+)
+
+// buildCondition creates a Like condition if a wildcard is present. Otherwise Eq is used.
+func buildCondition(name, value string) builder.Cond {
+	if strings.Contains(value, "*") {
+		return builder.Like{name, strings.ReplaceAll(strings.ReplaceAll(value, "_", "\\_"), "*", "%")}
+	}
+	return builder.Eq{name: value}
+}
+
+type RecipeSearchOptions struct {
+	OwnerID int64
+	Name    string
+	Version string
+	User    string
+	Channel string
+}
+
+// SearchRecipes gets all recipes matching the search options
+func SearchRecipes(ctx context.Context, opts *RecipeSearchOptions) ([]string, error) {
+	var cond builder.Cond = builder.Eq{
+		"package_file.is_lead":        true,
+		"package.type":                packages.TypeConan,
+		"package.owner_id":            opts.OwnerID,
+		"package_version.is_internal": false,
+	}
+
+	if opts.Name != "" {
+		cond = cond.And(buildCondition("package.lower_name", strings.ToLower(opts.Name)))
+	}
+	if opts.Version != "" {
+		cond = cond.And(buildCondition("package_version.lower_version", strings.ToLower(opts.Version)))
+	}
+	if opts.User != "" || opts.Channel != "" {
+		var propsCond builder.Cond = builder.Eq{
+			"package_property.ref_type": packages.PropertyTypeFile,
+		}
+		propsCond = propsCond.And(builder.Expr("package_property.ref_id = package_file.id"))
+
+		count := 0
+		propsCondBlock := builder.NewCond()
+		if opts.User != "" {
+			count++
+			propsCondBlock = propsCondBlock.Or(builder.Eq{"package_property.name": conan_module.PropertyRecipeUser}.And(buildCondition("package_property.value", opts.User)))
+		}
+		if opts.Channel != "" {
+			count++
+			propsCondBlock = propsCondBlock.Or(builder.Eq{"package_property.name": conan_module.PropertyRecipeChannel}.And(buildCondition("package_property.value", opts.Channel)))
+		}
+		propsCond = propsCond.And(propsCondBlock)
+
+		cond = cond.And(builder.Eq{
+			strconv.Itoa(count): builder.Select("COUNT(*)").Where(propsCond).From("package_property"),
+		})
+	}
+
+	query := builder.
+		Select("package.name, package_version.version, package_file.id").
+		From("package_file").
+		Join("INNER", "package_version", "package_version.id = package_file.version_id").
+		Join("INNER", "package", "package.id = package_version.package_id").
+		Where(cond)
+
+	results := make([]struct {
+		Name    string
+		Version string
+		ID      int64
+	}, 0, 5)
+	err := db.GetEngine(ctx).SQL(query).Find(&results)
+	if err != nil {
+		return nil, err
+	}
+
+	unique := make(map[string]bool)
+	for _, info := range results {
+		recipe := fmt.Sprintf("%s/%s", info.Name, info.Version)
+
+		props, _ := packages.GetProperties(ctx, packages.PropertyTypeFile, info.ID)
+		if len(props) > 0 {
+			var (
+				user    = ""
+				channel = ""
+			)
+			for _, prop := range props {
+				if prop.Name == conan_module.PropertyRecipeUser {
+					user = prop.Value
+				}
+				if prop.Name == conan_module.PropertyRecipeChannel {
+					channel = prop.Value
+				}
+			}
+			if user != "" && channel != "" {
+				recipe = fmt.Sprintf("%s@%s/%s", recipe, user, channel)
+			}
+		}
+
+		unique[recipe] = true
+	}
+
+	recipes := make([]string, 0, len(unique))
+	for recipe := range unique {
+		recipes = append(recipes, recipe)
+	}
+	return recipes, nil
+}
+
+// GetPackageInfo gets the Conaninfo for a package
+func GetPackageInfo(ctx context.Context, ownerID int64, ref *conan_module.PackageReference) (string, error) {
+	values, err := findPropertyValues(
+		ctx,
+		conan_module.PropertyPackageInfo,
+		ownerID,
+		ref.Recipe.Name,
+		ref.Recipe.Version,
+		map[string]string{
+			conan_module.PropertyRecipeUser:       ref.Recipe.User,
+			conan_module.PropertyRecipeChannel:    ref.Recipe.Channel,
+			conan_module.PropertyRecipeRevision:   ref.Recipe.Revision,
+			conan_module.PropertyPackageReference: ref.Reference,
+			conan_module.PropertyPackageRevision:  ref.Revision,
+		},
+	)
+	if err != nil {
+		return "", err
+	}
+
+	if len(values) == 0 {
+		return "", ErrPackageReferenceNotExist
+	}
+
+	return values[0].Value, nil
+}