From 1d332342db6d5bd4e1552d8d46720bf1b948c26b Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 30 Mar 2022 10:42:47 +0200 Subject: Add Package Registry (#16510) * Added package store settings. * Added models. * Added generic package registry. * Added tests. * Added NuGet package registry. * Moved service index to api file. * Added NPM package registry. * Added Maven package registry. * Added PyPI package registry. * Summary is deprecated. * Changed npm name. * Sanitize project url. * Allow only scoped packages. * Added user interface. * Changed method name. * Added missing migration file. * Set page info. * Added documentation. * Added documentation links. * Fixed wrong error message. * Lint template files. * Fixed merge errors. * Fixed unit test storage path. * Switch to json module. * Added suggestions. * Added package webhook. * Add package api. * Fixed swagger file. * Fixed enum and comments. * Fixed NuGet pagination. * Print test names. * Added api tests. * Fixed access level. * Fix User unmarshal. * Added RubyGems package registry. * Fix lint. * Implemented io.Writer. * Added support for sha256/sha512 checksum files. * Improved maven-metadata.xml support. * Added support for symbol package uploads. * Added tests. * Added overview docs. * Added npm dependencies and keywords. * Added no-packages information. * Display file size. * Display asset count. * Fixed filter alignment. * Added package icons. * Formatted instructions. * Allow anonymous package downloads. * Fixed comments. * Fixed postgres test. * Moved file. * Moved models to models/packages. * Use correct error response format per client. * Use simpler search form. * Fixed IsProd. * Restructured data model. * Prevent empty filename. * Fix swagger. * Implemented user/org registry. * Implemented UI. * Use GetUserByIDCtx. * Use table for dependencies. * make svg * Added support for unscoped npm packages. * Add support for npm dist tags. * Added tests for npm tags. * Unlink packages if repository gets deleted. * Prevent user/org delete if a packages exist. * Use package unlink in repository service. * Added support for composer packages. * Restructured package docs. * Added missing tests. * Fixed generic content page. * Fixed docs. * Fixed swagger. * Added missing type. * Fixed ambiguous column. * Organize content store by sha256 hash. * Added admin package management. * Added support for sorting. * Add support for multiple identical versions/files. * Added missing repository unlink. * Added file properties. * make fmt * lint * Added Conan package registry. * Updated docs. * Unify package names. * Added swagger enum. * Use longer TEXT column type. * Removed version composite key. * Merged package and container registry. * Removed index. * Use dedicated package router. * Moved files to new location. * Updated docs. * Fixed JOIN order. * Fixed GROUP BY statement. * Fixed GROUP BY #2. * Added symbol server support. * Added more tests. * Set NOT NULL. * Added setting to disable package registries. * Moved auth into service. * refactor * Use ctx everywhere. * Added package cleanup task. * Changed packages path. * Added container registry. * Refactoring * Updated comparison. * Fix swagger. * Fixed table order. * Use token auth for npm routes. * Enabled ReverseProxy auth. * Added packages link for orgs. * Fixed anonymous org access. * Enable copy button for setup instructions. * Merge error * Added suggestions. * Fixed merge. * Handle "generic". * Added link for TODO. * Added suggestions. * Changed temporary buffer filename. * Added suggestions. * Apply suggestions from code review Co-authored-by: Thomas Boerger * Update docs/content/doc/packages/nuget.en-us.md Co-authored-by: wxiaoguang Co-authored-by: Thomas Boerger --- modules/packages/npm/creator.go | 256 +++++++++++++++++++++++++++++++++ modules/packages/npm/creator_test.go | 272 +++++++++++++++++++++++++++++++++++ modules/packages/npm/metadata.go | 24 ++++ 3 files changed, 552 insertions(+) create mode 100644 modules/packages/npm/creator.go create mode 100644 modules/packages/npm/creator_test.go create mode 100644 modules/packages/npm/metadata.go (limited to 'modules/packages/npm') diff --git a/modules/packages/npm/creator.go b/modules/packages/npm/creator.go new file mode 100644 index 0000000000..88ce55ecdb --- /dev/null +++ b/modules/packages/npm/creator.go @@ -0,0 +1,256 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package npm + +import ( + "bytes" + "crypto/sha1" + "crypto/sha512" + "encoding/base64" + "errors" + "fmt" + "io" + "regexp" + "strings" + "time" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/validation" + + "github.com/hashicorp/go-version" +) + +var ( + // ErrInvalidPackage indicates an invalid package + ErrInvalidPackage = errors.New("The package is invalid") + // ErrInvalidPackageName indicates an invalid name + ErrInvalidPackageName = errors.New("The package name is invalid") + // ErrInvalidPackageVersion indicates an invalid version + ErrInvalidPackageVersion = errors.New("The package version is invalid") + // ErrInvalidAttachment indicates a invalid attachment + ErrInvalidAttachment = errors.New("The package attachment is invalid") + // ErrInvalidIntegrity indicates an integrity validation error + ErrInvalidIntegrity = errors.New("Failed to validate integrity") +) + +var nameMatch = regexp.MustCompile(`\A((@[^\s\/~'!\(\)\*]+?)[\/])?([^_.][^\s\/~'!\(\)\*]+)\z`) + +// Package represents a npm package +type Package struct { + Name string + Version string + DistTags []string + Metadata Metadata + Filename string + Data []byte +} + +// PackageMetadata https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#package +type PackageMetadata struct { + ID string `json:"_id"` + Name string `json:"name"` + Description string `json:"description"` + DistTags map[string]string `json:"dist-tags,omitempty"` + Versions map[string]*PackageMetadataVersion `json:"versions"` + Readme string `json:"readme,omitempty"` + Maintainers []User `json:"maintainers,omitempty"` + Time map[string]time.Time `json:"time,omitempty"` + Homepage string `json:"homepage,omitempty"` + Keywords []string `json:"keywords,omitempty"` + Repository Repository `json:"repository,omitempty"` + Author User `json:"author"` + ReadmeFilename string `json:"readmeFilename,omitempty"` + Users map[string]bool `json:"users,omitempty"` + License string `json:"license,omitempty"` +} + +// PackageMetadataVersion https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version +type PackageMetadataVersion struct { + ID string `json:"_id"` + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Author User `json:"author"` + Homepage string `json:"homepage,omitempty"` + License string `json:"license,omitempty"` + Repository Repository `json:"repository,omitempty"` + Keywords []string `json:"keywords,omitempty"` + Dependencies map[string]string `json:"dependencies,omitempty"` + DevDependencies map[string]string `json:"devDependencies,omitempty"` + PeerDependencies map[string]string `json:"peerDependencies,omitempty"` + OptionalDependencies map[string]string `json:"optionalDependencies,omitempty"` + Readme string `json:"readme,omitempty"` + Dist PackageDistribution `json:"dist"` + Maintainers []User `json:"maintainers,omitempty"` +} + +// PackageDistribution https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version +type PackageDistribution struct { + Integrity string `json:"integrity"` + Shasum string `json:"shasum"` + Tarball string `json:"tarball"` + FileCount int `json:"fileCount,omitempty"` + UnpackedSize int `json:"unpackedSize,omitempty"` + NpmSignature string `json:"npm-signature,omitempty"` +} + +// User https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#package +type User struct { + Username string `json:"username,omitempty"` + Name string `json:"name"` + Email string `json:"email,omitempty"` + URL string `json:"url,omitempty"` +} + +// UnmarshalJSON is needed because User objects can be strings or objects +func (u *User) UnmarshalJSON(data []byte) error { + switch data[0] { + case '"': + if err := json.Unmarshal(data, &u.Name); err != nil { + return err + } + case '{': + var tmp struct { + Username string `json:"username"` + Name string `json:"name"` + Email string `json:"email"` + URL string `json:"url"` + } + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + u.Username = tmp.Username + u.Name = tmp.Name + u.Email = tmp.Email + u.URL = tmp.URL + } + return nil +} + +// Repository https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version +type Repository struct { + Type string `json:"type"` + URL string `json:"url"` +} + +// PackageAttachment https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#package +type PackageAttachment struct { + ContentType string `json:"content_type"` + Data string `json:"data"` + Length int `json:"length"` +} + +type packageUpload struct { + PackageMetadata + Attachments map[string]*PackageAttachment `json:"_attachments"` +} + +// ParsePackage parses the content into a npm package +func ParsePackage(r io.Reader) (*Package, error) { + var upload packageUpload + if err := json.NewDecoder(r).Decode(&upload); err != nil { + return nil, err + } + + for _, meta := range upload.Versions { + if !validateName(meta.Name) { + return nil, ErrInvalidPackageName + } + + v, err := version.NewSemver(meta.Version) + if err != nil { + return nil, ErrInvalidPackageVersion + } + + scope := "" + name := meta.Name + nameParts := strings.SplitN(meta.Name, "/", 2) + if len(nameParts) == 2 { + scope = nameParts[0] + name = nameParts[1] + } + + if !validation.IsValidURL(meta.Homepage) { + meta.Homepage = "" + } + + p := &Package{ + Name: meta.Name, + Version: v.String(), + DistTags: make([]string, 0, 1), + Metadata: Metadata{ + Scope: scope, + Name: name, + Description: meta.Description, + Author: meta.Author.Name, + License: meta.License, + ProjectURL: meta.Homepage, + Keywords: meta.Keywords, + Dependencies: meta.Dependencies, + DevelopmentDependencies: meta.DevDependencies, + PeerDependencies: meta.PeerDependencies, + OptionalDependencies: meta.OptionalDependencies, + Readme: meta.Readme, + }, + } + + for tag := range upload.DistTags { + p.DistTags = append(p.DistTags, tag) + } + + p.Filename = strings.ToLower(fmt.Sprintf("%s-%s.tgz", name, p.Version)) + + attachment := func() *PackageAttachment { + for _, a := range upload.Attachments { + return a + } + return nil + }() + if attachment == nil || len(attachment.Data) == 0 { + return nil, ErrInvalidAttachment + } + + data, err := base64.StdEncoding.DecodeString(attachment.Data) + if err != nil { + return nil, ErrInvalidAttachment + } + p.Data = data + + integrity := strings.SplitN(meta.Dist.Integrity, "-", 2) + if len(integrity) != 2 { + return nil, ErrInvalidIntegrity + } + integrityHash, err := base64.StdEncoding.DecodeString(integrity[1]) + if err != nil { + return nil, ErrInvalidIntegrity + } + var hash []byte + switch integrity[0] { + case "sha1": + tmp := sha1.Sum(data) + hash = tmp[:] + case "sha512": + tmp := sha512.Sum512(data) + hash = tmp[:] + } + if !bytes.Equal(integrityHash, hash) { + return nil, ErrInvalidIntegrity + } + + return p, nil + } + + return nil, ErrInvalidPackage +} + +func validateName(name string) bool { + if strings.TrimSpace(name) != name { + return false + } + if len(name) == 0 || len(name) > 214 { + return false + } + return nameMatch.MatchString(name) +} diff --git a/modules/packages/npm/creator_test.go b/modules/packages/npm/creator_test.go new file mode 100644 index 0000000000..64ae6238f3 --- /dev/null +++ b/modules/packages/npm/creator_test.go @@ -0,0 +1,272 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package npm + +import ( + "bytes" + "encoding/base64" + "fmt" + "strings" + "testing" + + "code.gitea.io/gitea/modules/json" + + "github.com/stretchr/testify/assert" +) + +func TestParsePackage(t *testing.T) { + packageScope := "@scope" + packageName := "test-package" + packageFullName := packageScope + "/" + packageName + packageVersion := "1.0.1-pre" + packageTag := "latest" + packageAuthor := "KN4CK3R" + packageDescription := "Test Description" + data := "H4sIAAAAAAAA/ytITM5OTE/VL4DQelnF+XkMVAYGBgZmJiYK2MRBwNDcSIHB2NTMwNDQzMwAqA7IMDUxA9LUdgg2UFpcklgEdAql5kD8ogCnhwio5lJQUMpLzE1VslJQcihOzi9I1S9JLS7RhSYIJR2QgrLUouLM/DyQGkM9Az1D3YIiqExKanFyUWZBCVQ2BKhVwQVJDKwosbQkI78IJO/tZ+LsbRykxFXLNdA+HwWjYBSMgpENACgAbtAACAAA" + integrity := "sha512-yA4FJsVhetynGfOC1jFf79BuS+jrHbm0fhh+aHzCQkOaOBXKf9oBnC4a6DnLLnEsHQDRLYd00cwj8sCXpC+wIg==" + + t.Run("InvalidUpload", func(t *testing.T) { + p, err := ParsePackage(bytes.NewReader([]byte{0})) + assert.Nil(t, p) + assert.Error(t, err) + }) + + t.Run("InvalidUploadNoData", func(t *testing.T) { + b, _ := json.Marshal(packageUpload{}) + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidPackage) + }) + + t.Run("InvalidPackageName", func(t *testing.T) { + test := func(t *testing.T, name string) { + b, _ := json.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: name, + Name: name, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: name, + }, + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidPackageName) + } + + test(t, " test ") + test(t, " test") + test(t, "test ") + test(t, "te st") + test(t, "invalid/scope") + test(t, "@invalid/_name") + test(t, "@invalid/.name") + }) + + t.Run("ValidPackageName", func(t *testing.T) { + test := func(t *testing.T, name string) { + b, _ := json.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: name, + Name: name, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: name, + }, + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidPackageVersion) + } + + test(t, "test") + test(t, "@scope/name") + test(t, packageFullName) + }) + + t.Run("InvalidPackageVersion", func(t *testing.T) { + version := "first-version" + b, _ := json.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: packageFullName, + Name: packageFullName, + Versions: map[string]*PackageMetadataVersion{ + version: { + Name: packageFullName, + Version: version, + }, + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidPackageVersion) + }) + + t.Run("InvalidAttachment", func(t *testing.T) { + b, _ := json.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: packageFullName, + Name: packageFullName, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: packageFullName, + Version: packageVersion, + }, + }, + }, + Attachments: map[string]*PackageAttachment{ + "dummy.tgz": {}, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidAttachment) + }) + + t.Run("InvalidData", func(t *testing.T) { + filename := fmt.Sprintf("%s-%s.tgz", packageFullName, packageVersion) + b, _ := json.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: packageFullName, + Name: packageFullName, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: packageFullName, + Version: packageVersion, + }, + }, + }, + Attachments: map[string]*PackageAttachment{ + filename: { + Data: "/", + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidAttachment) + }) + + t.Run("InvalidIntegrity", func(t *testing.T) { + filename := fmt.Sprintf("%s-%s.tgz", packageFullName, packageVersion) + b, _ := json.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: packageFullName, + Name: packageFullName, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: packageFullName, + Version: packageVersion, + Dist: PackageDistribution{ + Integrity: "sha512-test==", + }, + }, + }, + }, + Attachments: map[string]*PackageAttachment{ + filename: { + Data: data, + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidIntegrity) + }) + + t.Run("InvalidIntegrity2", func(t *testing.T) { + filename := fmt.Sprintf("%s-%s.tgz", packageFullName, packageVersion) + b, _ := json.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: packageFullName, + Name: packageFullName, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: packageFullName, + Version: packageVersion, + Dist: PackageDistribution{ + Integrity: integrity, + }, + }, + }, + }, + Attachments: map[string]*PackageAttachment{ + filename: { + Data: base64.StdEncoding.EncodeToString([]byte("data")), + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidIntegrity) + }) + + t.Run("Valid", func(t *testing.T) { + filename := fmt.Sprintf("%s-%s.tgz", packageFullName, packageVersion) + b, _ := json.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: packageFullName, + Name: packageFullName, + DistTags: map[string]string{ + packageTag: packageVersion, + }, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: packageFullName, + Version: packageVersion, + Description: packageDescription, + Author: User{Name: packageAuthor}, + License: "MIT", + Homepage: "https://gitea.io/", + Readme: packageDescription, + Dependencies: map[string]string{ + "package": "1.2.0", + }, + Dist: PackageDistribution{ + Integrity: integrity, + }, + }, + }, + }, + Attachments: map[string]*PackageAttachment{ + filename: { + Data: data, + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.NotNil(t, p) + assert.NoError(t, err) + + assert.Equal(t, packageFullName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.Equal(t, []string{packageTag}, p.DistTags) + assert.Equal(t, fmt.Sprintf("%s-%s.tgz", strings.Split(packageFullName, "/")[1], packageVersion), p.Filename) + b, _ = base64.StdEncoding.DecodeString(data) + assert.Equal(t, b, p.Data) + assert.Equal(t, packageName, p.Metadata.Name) + assert.Equal(t, packageScope, p.Metadata.Scope) + assert.Equal(t, packageDescription, p.Metadata.Description) + assert.Equal(t, packageDescription, p.Metadata.Readme) + assert.Equal(t, packageAuthor, p.Metadata.Author) + assert.Equal(t, "MIT", p.Metadata.License) + assert.Equal(t, "https://gitea.io/", p.Metadata.ProjectURL) + assert.Contains(t, p.Metadata.Dependencies, "package") + assert.Equal(t, "1.2.0", p.Metadata.Dependencies["package"]) + }) +} diff --git a/modules/packages/npm/metadata.go b/modules/packages/npm/metadata.go new file mode 100644 index 0000000000..643a4d344b --- /dev/null +++ b/modules/packages/npm/metadata.go @@ -0,0 +1,24 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package npm + +// TagProperty is the name of the property for tag management +const TagProperty = "npm.tag" + +// Metadata represents the metadata of a npm package +type Metadata struct { + Scope string `json:"scope,omitempty"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Author string `json:"author,omitempty"` + License string `json:"license,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + Keywords []string `json:"keywords,omitempty"` + Dependencies map[string]string `json:"dependencies,omitempty"` + DevelopmentDependencies map[string]string `json:"development_dependencies,omitempty"` + PeerDependencies map[string]string `json:"peer_dependencies,omitempty"` + OptionalDependencies map[string]string `json:"optional_dependencies,omitempty"` + Readme string `json:"readme,omitempty"` +} -- cgit v1.2.3