Add Package Registry (#16510)

* Added package store settings.

* Added models.

* Added generic package registry.

* Added tests.

* Added NuGet package registry.

* Moved service index to api file.

* Added NPM package registry.

* Added Maven package registry.

* Added PyPI package registry.

* Summary is deprecated.

* Changed npm name.

* Sanitize project url.

* Allow only scoped packages.

* Added user interface.

* Changed method name.

* Added missing migration file.

* Set page info.

* Added documentation.

* Added documentation links.

* Fixed wrong error message.

* Lint template files.

* Fixed merge errors.

* Fixed unit test storage path.

* Switch to json module.

* Added suggestions.

* Added package webhook.

* Add package api.

* Fixed swagger file.

* Fixed enum and comments.

* Fixed NuGet pagination.

* Print test names.

* Added api tests.

* Fixed access level.

* Fix User unmarshal.

* Added RubyGems package registry.

* Fix lint.

* Implemented io.Writer.

* Added support for sha256/sha512 checksum files.

* Improved maven-metadata.xml support.

* Added support for symbol package uploads.

* Added tests.

* Added overview docs.

* Added npm dependencies and keywords.

* Added no-packages information.

* Display file size.

* Display asset count.

* Fixed filter alignment.

* Added package icons.

* Formatted instructions.

* Allow anonymous package downloads.

* Fixed comments.

* Fixed postgres test.

* Moved file.

* Moved models to models/packages.

* Use correct error response format per client.

* Use simpler search form.

* Fixed IsProd.

* Restructured data model.

* Prevent empty filename.

* Fix swagger.

* Implemented user/org registry.

* Implemented UI.

* Use GetUserByIDCtx.

* Use table for dependencies.

* make svg

* Added support for unscoped npm packages.

* Add support for npm dist tags.

* Added tests for npm tags.

* Unlink packages if repository gets deleted.

* Prevent user/org delete if a packages exist.

* Use package unlink in repository service.

* Added support for composer packages.

* Restructured package docs.

* Added missing tests.

* Fixed generic content page.

* Fixed docs.

* Fixed swagger.

* Added missing type.

* Fixed ambiguous column.

* Organize content store by sha256 hash.

* Added admin package management.

* Added support for sorting.

* Add support for multiple identical versions/files.

* Added missing repository unlink.

* Added file properties.

* make fmt

* lint

* Added Conan package registry.

* Updated docs.

* Unify package names.

* Added swagger enum.

* Use longer TEXT column type.

* Removed version composite key.

* Merged package and container registry.

* Removed index.

* Use dedicated package router.

* Moved files to new location.

* Updated docs.

* Fixed JOIN order.

* Fixed GROUP BY statement.

* Fixed GROUP BY #2.

* Added symbol server support.

* Added more tests.

* Set NOT NULL.

* Added setting to disable package registries.

* Moved auth into service.

* refactor

* Use ctx everywhere.

* Added package cleanup task.

* Changed packages path.

* Added container registry.

* Refactoring

* Updated comparison.

* Fix swagger.

* Fixed table order.

* Use token auth for npm routes.

* Enabled ReverseProxy auth.

* Added packages link for orgs.

* Fixed anonymous org access.

* Enable copy button for setup instructions.

* Merge error

* Added suggestions.

* Fixed merge.

* Handle "generic".

* Added link for TODO.

* Added suggestions.

* Changed temporary buffer filename.

* Added suggestions.

* Apply suggestions from code review

Co-authored-by: Thomas Boerger <thomas@webhippie.de>

* Update docs/content/doc/packages/nuget.en-us.md

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: Thomas Boerger <thomas@webhippie.de>

2 files changed, 434 insertions, 0 deletions

diff --git a/routers/api/packages/maven/api.go b/routers/api/packages/maven/api.go
new file mode 100644
index 0000000000..b60a317814
--- /dev/null
+++ b/routers/api/packages/maven/api.go
@@ -0,0 +1,56 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package maven
+
+import (
+	"encoding/xml"
+	"sort"
+	"strings"
+
+	packages_model "code.gitea.io/gitea/models/packages"
+	maven_module "code.gitea.io/gitea/modules/packages/maven"
+)
+
+// MetadataResponse https://maven.apache.org/ref/3.2.5/maven-repository-metadata/repository-metadata.html
+type MetadataResponse struct {
+	XMLName    xml.Name `xml:"metadata"`
+	GroupID    string   `xml:"groupId"`
+	ArtifactID string   `xml:"artifactId"`
+	Release    string   `xml:"versioning>release,omitempty"`
+	Latest     string   `xml:"versioning>latest"`
+	Version    []string `xml:"versioning>versions>version"`
+}
+
+func createMetadataResponse(pds []*packages_model.PackageDescriptor) *MetadataResponse {
+	sort.Slice(pds, func(i, j int) bool {
+		// Maven and Gradle order packages by their creation timestamp and not by their version string
+		return pds[i].Version.CreatedUnix < pds[j].Version.CreatedUnix
+	})
+
+	var release *packages_model.PackageDescriptor
+
+	versions := make([]string, 0, len(pds))
+	for _, pd := range pds {
+		if !strings.HasSuffix(pd.Version.Version, "-SNAPSHOT") {
+			release = pd
+		}
+		versions = append(versions, pd.Version.Version)
+	}
+
+	latest := pds[len(pds)-1]
+
+	metadata := latest.Metadata.(*maven_module.Metadata)
+
+	resp := &MetadataResponse{
+		GroupID:    metadata.GroupID,
+		ArtifactID: metadata.ArtifactID,
+		Latest:     latest.Version.Version,
+		Version:    versions,
+	}
+	if release != nil {
+		resp.Release = release.Version.Version
+	}
+	return resp
+}
diff --git a/routers/api/packages/maven/maven.go b/routers/api/packages/maven/maven.go
new file mode 100644
index 0000000000..bba4babf04
--- /dev/null
+++ b/routers/api/packages/maven/maven.go
@@ -0,0 +1,378 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package maven
+
+import (
+	"crypto/md5"
+	"crypto/sha1"
+	"crypto/sha256"
+	"crypto/sha512"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	packages_model "code.gitea.io/gitea/models/packages"
+	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/json"
+	"code.gitea.io/gitea/modules/log"
+	packages_module "code.gitea.io/gitea/modules/packages"
+	maven_module "code.gitea.io/gitea/modules/packages/maven"
+	"code.gitea.io/gitea/routers/api/packages/helper"
+	packages_service "code.gitea.io/gitea/services/packages"
+)
+
+const (
+	mavenMetadataFile = "maven-metadata.xml"
+	extensionMD5      = ".md5"
+	extensionSHA1     = ".sha1"
+	extensionSHA256   = ".sha256"
+	extensionSHA512   = ".sha512"
+)
+
+var (
+	errInvalidParameters = errors.New("request parameters are invalid")
+	illegalCharacters    = regexp.MustCompile(`[\\/:"<>|?\*]`)
+)
+
+func apiError(ctx *context.Context, status int, obj interface{}) {
+	helper.LogAndProcessError(ctx, status, obj, func(message string) {
+		ctx.PlainText(status, message)
+	})
+}
+
+// DownloadPackageFile serves the content of a package
+func DownloadPackageFile(ctx *context.Context) {
+	params, err := extractPathParameters(ctx)
+	if err != nil {
+		apiError(ctx, http.StatusBadRequest, err)
+		return
+	}
+
+	if params.IsMeta && params.Version == "" {
+		serveMavenMetadata(ctx, params)
+	} else {
+		servePackageFile(ctx, params)
+	}
+}
+
+func serveMavenMetadata(ctx *context.Context, params parameters) {
+	// /com/foo/project/maven-metadata.xml[.md5/.sha1/.sha256/.sha512]
+
+	packageName := params.GroupID + "-" + params.ArtifactID
+	pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeMaven, packageName)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	if len(pvs) == 0 {
+		apiError(ctx, http.StatusNotFound, packages_model.ErrPackageNotExist)
+		return
+	}
+
+	pds, err := packages_model.GetPackageDescriptors(ctx, pvs)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	xmlMetadata, err := xml.Marshal(createMetadataResponse(pds))
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	xmlMetadataWithHeader := append([]byte(xml.Header), xmlMetadata...)
+
+	ext := strings.ToLower(filepath.Ext(params.Filename))
+	if isChecksumExtension(ext) {
+		var hash []byte
+		switch ext {
+		case extensionMD5:
+			tmp := md5.Sum(xmlMetadataWithHeader)
+			hash = tmp[:]
+		case extensionSHA1:
+			tmp := sha1.Sum(xmlMetadataWithHeader)
+			hash = tmp[:]
+		case extensionSHA256:
+			tmp := sha256.Sum256(xmlMetadataWithHeader)
+			hash = tmp[:]
+		case extensionSHA512:
+			tmp := sha512.Sum512(xmlMetadataWithHeader)
+			hash = tmp[:]
+		}
+		ctx.PlainText(http.StatusOK, fmt.Sprintf("%x", hash))
+		return
+	}
+
+	ctx.PlainTextBytes(http.StatusOK, xmlMetadataWithHeader)
+}
+
+func servePackageFile(ctx *context.Context, params parameters) {
+	packageName := params.GroupID + "-" + params.ArtifactID
+
+	pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeMaven, packageName, params.Version)
+	if err != nil {
+		if err == packages_model.ErrPackageNotExist {
+			apiError(ctx, http.StatusNotFound, err)
+		} else {
+			apiError(ctx, http.StatusInternalServerError, err)
+		}
+		return
+	}
+
+	filename := params.Filename
+
+	ext := strings.ToLower(filepath.Ext(filename))
+	if isChecksumExtension(ext) {
+		filename = filename[:len(filename)-len(ext)]
+	}
+
+	pf, err := packages_model.GetFileForVersionByName(ctx, pv.ID, filename, packages_model.EmptyFileKey)
+	if err != nil {
+		if err == packages_model.ErrPackageFileNotExist {
+			apiError(ctx, http.StatusNotFound, err)
+		} else {
+			apiError(ctx, http.StatusInternalServerError, err)
+		}
+		return
+	}
+
+	pb, err := packages_model.GetBlobByID(ctx, pf.BlobID)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	if isChecksumExtension(ext) {
+		var hash string
+		switch ext {
+		case extensionMD5:
+			hash = pb.HashMD5
+		case extensionSHA1:
+			hash = pb.HashSHA1
+		case extensionSHA256:
+			hash = pb.HashSHA256
+		case extensionSHA512:
+			hash = pb.HashSHA512
+		}
+		ctx.PlainText(http.StatusOK, hash)
+		return
+	}
+
+	s, err := packages_module.NewContentStore().Get(packages_module.BlobHash256Key(pb.HashSHA256))
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+	}
+	defer s.Close()
+
+	if pf.IsLead {
+		if err := packages_model.IncrementDownloadCounter(ctx, pv.ID); err != nil {
+			log.Error("Error incrementing download counter: %v", err)
+		}
+	}
+
+	ctx.ServeStream(s, pf.Name)
+}
+
+// UploadPackageFile adds a file to the package. If the package does not exist, it gets created.
+func UploadPackageFile(ctx *context.Context) {
+	params, err := extractPathParameters(ctx)
+	if err != nil {
+		apiError(ctx, http.StatusBadRequest, err)
+		return
+	}
+
+	log.Trace("Parameters: %+v", params)
+
+	// Ignore the package index /<name>/maven-metadata.xml
+	if params.IsMeta && params.Version == "" {
+		ctx.Status(http.StatusOK)
+		return
+	}
+
+	packageName := params.GroupID + "-" + params.ArtifactID
+
+	buf, err := packages_module.CreateHashedBufferFromReader(ctx.Req.Body, 32*1024*1024)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	defer buf.Close()
+
+	pvci := &packages_service.PackageCreationInfo{
+		PackageInfo: packages_service.PackageInfo{
+			Owner:       ctx.Package.Owner,
+			PackageType: packages_model.TypeMaven,
+			Name:        packageName,
+			Version:     params.Version,
+		},
+		SemverCompatible: false,
+		Creator:          ctx.Doer,
+	}
+
+	ext := filepath.Ext(params.Filename)
+
+	// Do not upload checksum files but compare the hashes.
+	if isChecksumExtension(ext) {
+		pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvci.Owner.ID, pvci.PackageType, pvci.Name, pvci.Version)
+		if err != nil {
+			if err == packages_model.ErrPackageNotExist {
+				apiError(ctx, http.StatusNotFound, err)
+				return
+			}
+			apiError(ctx, http.StatusInternalServerError, err)
+			return
+		}
+		pf, err := packages_model.GetFileForVersionByName(ctx, pv.ID, params.Filename[:len(params.Filename)-len(ext)], packages_model.EmptyFileKey)
+		if err != nil {
+			if err == packages_model.ErrPackageFileNotExist {
+				apiError(ctx, http.StatusNotFound, err)
+				return
+			}
+			apiError(ctx, http.StatusInternalServerError, err)
+			return
+		}
+		pb, err := packages_model.GetBlobByID(ctx, pf.BlobID)
+		if err != nil {
+			apiError(ctx, http.StatusInternalServerError, err)
+			return
+		}
+
+		hash, err := io.ReadAll(buf)
+		if err != nil {
+			apiError(ctx, http.StatusInternalServerError, err)
+			return
+		}
+
+		if (ext == extensionMD5 && pb.HashMD5 != string(hash)) ||
+			(ext == extensionSHA1 && pb.HashSHA1 != string(hash)) ||
+			(ext == extensionSHA256 && pb.HashSHA256 != string(hash)) ||
+			(ext == extensionSHA512 && pb.HashSHA512 != string(hash)) {
+			apiError(ctx, http.StatusBadRequest, "hash mismatch")
+			return
+		}
+
+		ctx.Status(http.StatusOK)
+		return
+	}
+
+	pfci := &packages_service.PackageFileCreationInfo{
+		PackageFileInfo: packages_service.PackageFileInfo{
+			Filename: params.Filename,
+		},
+		Data:   buf,
+		IsLead: false,
+	}
+
+	// If it's the package pom file extract the metadata
+	if ext == ".pom" {
+		pfci.IsLead = true
+
+		var err error
+		pvci.Metadata, err = maven_module.ParsePackageMetaData(buf)
+		if err != nil {
+			log.Error("Error parsing package metadata: %v", err)
+		}
+
+		if pvci.Metadata != nil {
+			pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvci.Owner.ID, pvci.PackageType, pvci.Name, pvci.Version)
+			if err != nil && err != packages_model.ErrPackageNotExist {
+				apiError(ctx, http.StatusInternalServerError, err)
+				return
+			}
+			if pv != nil {
+				raw, err := json.Marshal(pvci.Metadata)
+				if err != nil {
+					apiError(ctx, http.StatusInternalServerError, err)
+					return
+				}
+				pv.MetadataJSON = string(raw)
+				if err := packages_model.UpdateVersion(ctx, pv); err != nil {
+					apiError(ctx, http.StatusInternalServerError, err)
+					return
+				}
+			}
+		}
+
+		if _, err := buf.Seek(0, io.SeekStart); err != nil {
+			apiError(ctx, http.StatusInternalServerError, err)
+			return
+		}
+	}
+
+	_, _, err = packages_service.CreatePackageOrAddFileToExisting(
+		pvci,
+		pfci,
+	)
+	if err != nil {
+		if err == packages_model.ErrDuplicatePackageFile {
+			apiError(ctx, http.StatusBadRequest, err)
+			return
+		}
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	ctx.Status(http.StatusCreated)
+}
+
+func isChecksumExtension(ext string) bool {
+	return ext == extensionMD5 || ext == extensionSHA1 || ext == extensionSHA256 || ext == extensionSHA512
+}
+
+type parameters struct {
+	GroupID    string
+	ArtifactID string
+	Version    string
+	Filename   string
+	IsMeta     bool
+}
+
+func extractPathParameters(ctx *context.Context) (parameters, error) {
+	parts := strings.Split(ctx.Params("*"), "/")
+
+	p := parameters{
+		Filename: parts[len(parts)-1],
+	}
+
+	p.IsMeta = p.Filename == mavenMetadataFile ||
+		p.Filename == mavenMetadataFile+extensionMD5 ||
+		p.Filename == mavenMetadataFile+extensionSHA1 ||
+		p.Filename == mavenMetadataFile+extensionSHA256 ||
+		p.Filename == mavenMetadataFile+extensionSHA512
+
+	parts = parts[:len(parts)-1]
+	if len(parts) == 0 {
+		return p, errInvalidParameters
+	}
+
+	p.Version = parts[len(parts)-1]
+	if p.IsMeta && !strings.HasSuffix(p.Version, "-SNAPSHOT") {
+		p.Version = ""
+	} else {
+		parts = parts[:len(parts)-1]
+	}
+
+	if illegalCharacters.MatchString(p.Version) {
+		return p, errInvalidParameters
+	}
+
+	if len(parts) < 2 {
+		return p, errInvalidParameters
+	}
+
+	p.ArtifactID = parts[len(parts)-1]
+	p.GroupID = strings.Join(parts[:len(parts)-1], ".")
+
+	if illegalCharacters.MatchString(p.GroupID) || illegalCharacters.MatchString(p.ArtifactID) {
+		return p, errInvalidParameters
+	}
+
+	return p, nil
+}