aboutsummaryrefslogtreecommitdiffstats
path: root/modules
diff options
context:
space:
mode:
authorKN4CK3R <admin@oldschoolhack.me>2022-03-30 10:42:47 +0200
committerGitHub <noreply@github.com>2022-03-30 16:42:47 +0800
commit1d332342db6d5bd4e1552d8d46720bf1b948c26b (patch)
treeca0c8931e5da85e71037ed43d7a90826ba708d9d /modules
parent2bce1ea9862c70ebb69963e65bb84dcad6ebb31c (diff)
downloadgitea-1d332342db6d5bd4e1552d8d46720bf1b948c26b.tar.gz
gitea-1d332342db6d5bd4e1552d8d46720bf1b948c26b.zip
Add Package Registry (#16510)
* Added package store settings. * Added models. * Added generic package registry. * Added tests. * Added NuGet package registry. * Moved service index to api file. * Added NPM package registry. * Added Maven package registry. * Added PyPI package registry. * Summary is deprecated. * Changed npm name. * Sanitize project url. * Allow only scoped packages. * Added user interface. * Changed method name. * Added missing migration file. * Set page info. * Added documentation. * Added documentation links. * Fixed wrong error message. * Lint template files. * Fixed merge errors. * Fixed unit test storage path. * Switch to json module. * Added suggestions. * Added package webhook. * Add package api. * Fixed swagger file. * Fixed enum and comments. * Fixed NuGet pagination. * Print test names. * Added api tests. * Fixed access level. * Fix User unmarshal. * Added RubyGems package registry. * Fix lint. * Implemented io.Writer. * Added support for sha256/sha512 checksum files. * Improved maven-metadata.xml support. * Added support for symbol package uploads. * Added tests. * Added overview docs. * Added npm dependencies and keywords. * Added no-packages information. * Display file size. * Display asset count. * Fixed filter alignment. * Added package icons. * Formatted instructions. * Allow anonymous package downloads. * Fixed comments. * Fixed postgres test. * Moved file. * Moved models to models/packages. * Use correct error response format per client. * Use simpler search form. * Fixed IsProd. * Restructured data model. * Prevent empty filename. * Fix swagger. * Implemented user/org registry. * Implemented UI. * Use GetUserByIDCtx. * Use table for dependencies. * make svg * Added support for unscoped npm packages. * Add support for npm dist tags. * Added tests for npm tags. * Unlink packages if repository gets deleted. * Prevent user/org delete if a packages exist. * Use package unlink in repository service. * Added support for composer packages. * Restructured package docs. * Added missing tests. * Fixed generic content page. * Fixed docs. * Fixed swagger. * Added missing type. * Fixed ambiguous column. * Organize content store by sha256 hash. * Added admin package management. * Added support for sorting. * Add support for multiple identical versions/files. * Added missing repository unlink. * Added file properties. * make fmt * lint * Added Conan package registry. * Updated docs. * Unify package names. * Added swagger enum. * Use longer TEXT column type. * Removed version composite key. * Merged package and container registry. * Removed index. * Use dedicated package router. * Moved files to new location. * Updated docs. * Fixed JOIN order. * Fixed GROUP BY statement. * Fixed GROUP BY #2. * Added symbol server support. * Added more tests. * Set NOT NULL. * Added setting to disable package registries. * Moved auth into service. * refactor * Use ctx everywhere. * Added package cleanup task. * Changed packages path. * Added container registry. * Refactoring * Updated comparison. * Fix swagger. * Fixed table order. * Use token auth for npm routes. * Enabled ReverseProxy auth. * Added packages link for orgs. * Fixed anonymous org access. * Enable copy button for setup instructions. * Merge error * Added suggestions. * Fixed merge. * Handle "generic". * Added link for TODO. * Added suggestions. * Changed temporary buffer filename. * Added suggestions. * Apply suggestions from code review Co-authored-by: Thomas Boerger <thomas@webhippie.de> * Update docs/content/doc/packages/nuget.en-us.md Co-authored-by: wxiaoguang <wxiaoguang@gmail.com> Co-authored-by: Thomas Boerger <thomas@webhippie.de>
Diffstat (limited to 'modules')
-rw-r--r--modules/context/context.go60
-rw-r--r--modules/context/package.go109
-rw-r--r--modules/convert/package.go43
-rw-r--r--modules/notification/base/notifier.go3
-rw-r--r--modules/notification/base/null.go9
-rw-r--r--modules/notification/notification.go15
-rw-r--r--modules/notification/webhook/webhook.go31
-rw-r--r--modules/packages/composer/metadata.go147
-rw-r--r--modules/packages/composer/metadata_test.go130
-rw-r--r--modules/packages/conan/conanfile_parser.go68
-rw-r--r--modules/packages/conan/conanfile_parser_test.go51
-rw-r--r--modules/packages/conan/conaninfo_parser.go123
-rw-r--r--modules/packages/conan/conaninfo_parser_test.go85
-rw-r--r--modules/packages/conan/metadata.go24
-rw-r--r--modules/packages/conan/reference.go155
-rw-r--r--modules/packages/conan/reference_test.go147
-rw-r--r--modules/packages/container/helm/helm.go56
-rw-r--r--modules/packages/container/metadata.go157
-rw-r--r--modules/packages/container/metadata_test.go62
-rw-r--r--modules/packages/container/oci/digest.go27
-rw-r--r--modules/packages/container/oci/mediatype.go36
-rw-r--r--modules/packages/container/oci/oci.go191
-rw-r--r--modules/packages/container/oci/reference.go17
-rw-r--r--modules/packages/content_store.go47
-rw-r--r--modules/packages/hashed_buffer.go70
-rw-r--r--modules/packages/maven/metadata.go89
-rw-r--r--modules/packages/maven/metadata_test.go73
-rw-r--r--modules/packages/multi_hasher.go123
-rw-r--r--modules/packages/multi_hasher_test.go54
-rw-r--r--modules/packages/npm/creator.go256
-rw-r--r--modules/packages/npm/creator_test.go272
-rw-r--r--modules/packages/npm/metadata.go24
-rw-r--r--modules/packages/nuget/metadata.go187
-rw-r--r--modules/packages/nuget/metadata_test.go163
-rw-r--r--modules/packages/nuget/symbol_extractor.go187
-rw-r--r--modules/packages/nuget/symbol_extractor_test.go82
-rw-r--r--modules/packages/pypi/metadata.go16
-rw-r--r--modules/packages/rubygems/marshal.go311
-rw-r--r--modules/packages/rubygems/marshal_test.go99
-rw-r--r--modules/packages/rubygems/metadata.go222
-rw-r--r--modules/packages/rubygems/metadata_test.go89
-rw-r--r--modules/setting/packages.go47
-rw-r--r--modules/setting/setting.go4
-rw-r--r--modules/storage/storage.go15
-rw-r--r--modules/structs/hook.go25
-rw-r--r--modules/structs/package.go33
-rw-r--r--modules/templates/helper.go12
-rw-r--r--modules/util/filebuffer/file_backed_buffer.go147
48 files changed, 4369 insertions, 24 deletions
diff --git a/modules/context/context.go b/modules/context/context.go
index eb0edef394..4905e1cb80 100644
--- a/modules/context/context.go
+++ b/modules/context/context.go
@@ -70,6 +70,7 @@ type Context struct {
ContextUser *user_model.User
Repo *Repository
Org *Organization
+ Package *Package
}
// TrHTMLEscapeArgs runs Tr but pre-escapes all arguments with html.EscapeString.
@@ -331,6 +332,18 @@ func (ctx *Context) RespHeader() http.Header {
return ctx.Resp.Header()
}
+// SetServeHeaders sets necessary content serve headers
+func (ctx *Context) SetServeHeaders(filename string) {
+ ctx.Resp.Header().Set("Content-Description", "File Transfer")
+ ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
+ ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+filename)
+ ctx.Resp.Header().Set("Content-Transfer-Encoding", "binary")
+ ctx.Resp.Header().Set("Expires", "0")
+ ctx.Resp.Header().Set("Cache-Control", "must-revalidate")
+ ctx.Resp.Header().Set("Pragma", "public")
+ ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition")
+}
+
// ServeContent serves content to http request
func (ctx *Context) ServeContent(name string, r io.ReadSeeker, params ...interface{}) {
modTime := time.Now()
@@ -340,14 +353,7 @@ func (ctx *Context) ServeContent(name string, r io.ReadSeeker, params ...interfa
modTime = v
}
}
- ctx.Resp.Header().Set("Content-Description", "File Transfer")
- ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
- ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+name)
- ctx.Resp.Header().Set("Content-Transfer-Encoding", "binary")
- ctx.Resp.Header().Set("Expires", "0")
- ctx.Resp.Header().Set("Cache-Control", "must-revalidate")
- ctx.Resp.Header().Set("Pragma", "public")
- ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition")
+ ctx.SetServeHeaders(name)
http.ServeContent(ctx.Resp, ctx.Req, name, modTime, r)
}
@@ -359,31 +365,41 @@ func (ctx *Context) ServeFile(file string, names ...string) {
} else {
name = path.Base(file)
}
- ctx.Resp.Header().Set("Content-Description", "File Transfer")
- ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
- ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+name)
- ctx.Resp.Header().Set("Content-Transfer-Encoding", "binary")
- ctx.Resp.Header().Set("Expires", "0")
- ctx.Resp.Header().Set("Cache-Control", "must-revalidate")
- ctx.Resp.Header().Set("Pragma", "public")
+ ctx.SetServeHeaders(name)
http.ServeFile(ctx.Resp, ctx.Req, file)
}
// ServeStream serves file via io stream
func (ctx *Context) ServeStream(rd io.Reader, name string) {
- ctx.Resp.Header().Set("Content-Description", "File Transfer")
- ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
- ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+name)
- ctx.Resp.Header().Set("Content-Transfer-Encoding", "binary")
- ctx.Resp.Header().Set("Expires", "0")
- ctx.Resp.Header().Set("Cache-Control", "must-revalidate")
- ctx.Resp.Header().Set("Pragma", "public")
+ ctx.SetServeHeaders(name)
_, err := io.Copy(ctx.Resp, rd)
if err != nil {
ctx.ServerError("Download file failed", err)
}
}
+// UploadStream returns the request body or the first form file
+// Only form files need to get closed.
+func (ctx *Context) UploadStream() (rd io.ReadCloser, needToClose bool, err error) {
+ contentType := strings.ToLower(ctx.Req.Header.Get("Content-Type"))
+ if strings.HasPrefix(contentType, "application/x-www-form-urlencoded") || strings.HasPrefix(contentType, "multipart/form-data") {
+ if err := ctx.Req.ParseMultipartForm(32 << 20); err != nil {
+ return nil, false, err
+ }
+ if ctx.Req.MultipartForm.File == nil {
+ return nil, false, http.ErrMissingFile
+ }
+ for _, files := range ctx.Req.MultipartForm.File {
+ if len(files) > 0 {
+ r, err := files[0].Open()
+ return r, true, err
+ }
+ }
+ return nil, false, http.ErrMissingFile
+ }
+ return ctx.Req.Body, false, nil
+}
+
// Error returned an error to web browser
func (ctx *Context) Error(status int, contents ...string) {
v := http.StatusText(status)
diff --git a/modules/context/package.go b/modules/context/package.go
new file mode 100644
index 0000000000..47af88c97b
--- /dev/null
+++ b/modules/context/package.go
@@ -0,0 +1,109 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package context
+
+import (
+ "fmt"
+ "net/http"
+
+ "code.gitea.io/gitea/models/organization"
+ packages_model "code.gitea.io/gitea/models/packages"
+ "code.gitea.io/gitea/models/perm"
+ user_model "code.gitea.io/gitea/models/user"
+)
+
+// Package contains owner, access mode and optional the package descriptor
+type Package struct {
+ Owner *user_model.User
+ AccessMode perm.AccessMode
+ Descriptor *packages_model.PackageDescriptor
+}
+
+// PackageAssignment returns a middleware to handle Context.Package assignment
+func PackageAssignment() func(ctx *Context) {
+ return func(ctx *Context) {
+ packageAssignment(ctx, func(status int, title string, obj interface{}) {
+ err, ok := obj.(error)
+ if !ok {
+ err = fmt.Errorf("%s", obj)
+ }
+ if status == http.StatusNotFound {
+ ctx.NotFound(title, err)
+ } else {
+ ctx.ServerError(title, err)
+ }
+ })
+ }
+}
+
+// PackageAssignmentAPI returns a middleware to handle Context.Package assignment
+func PackageAssignmentAPI() func(ctx *APIContext) {
+ return func(ctx *APIContext) {
+ packageAssignment(ctx.Context, ctx.Error)
+ }
+}
+
+func packageAssignment(ctx *Context, errCb func(int, string, interface{})) {
+ ctx.Package = &Package{
+ Owner: ctx.ContextUser,
+ }
+
+ if ctx.Doer != nil && ctx.Doer.ID == ctx.ContextUser.ID {
+ ctx.Package.AccessMode = perm.AccessModeOwner
+ } else {
+ if ctx.Package.Owner.IsOrganization() {
+ if organization.HasOrgOrUserVisible(ctx, ctx.Package.Owner, ctx.Doer) {
+ ctx.Package.AccessMode = perm.AccessModeRead
+ if ctx.Doer != nil {
+ var err error
+ ctx.Package.AccessMode, err = organization.OrgFromUser(ctx.Package.Owner).GetOrgUserMaxAuthorizeLevel(ctx.Doer.ID)
+ if err != nil {
+ errCb(http.StatusInternalServerError, "GetOrgUserMaxAuthorizeLevel", err)
+ return
+ }
+ }
+ }
+ } else {
+ ctx.Package.AccessMode = perm.AccessModeRead
+ }
+ }
+
+ packageType := ctx.Params("type")
+ name := ctx.Params("name")
+ version := ctx.Params("version")
+ if packageType != "" && name != "" && version != "" {
+ pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.Type(packageType), name, version)
+ if err != nil {
+ if err == packages_model.ErrPackageNotExist {
+ errCb(http.StatusNotFound, "GetVersionByNameAndVersion", err)
+ } else {
+ errCb(http.StatusInternalServerError, "GetVersionByNameAndVersion", err)
+ }
+ return
+ }
+
+ ctx.Package.Descriptor, err = packages_model.GetPackageDescriptor(ctx, pv)
+ if err != nil {
+ errCb(http.StatusInternalServerError, "GetPackageDescriptor", err)
+ return
+ }
+ }
+}
+
+// PackageContexter initializes a package context for a request.
+func PackageContexter() func(next http.Handler) http.Handler {
+ return func(next http.Handler) http.Handler {
+ return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+ ctx := Context{
+ Resp: NewResponse(resp),
+ Data: map[string]interface{}{},
+ }
+
+ ctx.Req = WithContext(req, &ctx)
+
+ next.ServeHTTP(ctx.Resp, ctx.Req)
+ })
+ }
+}
diff --git a/modules/convert/package.go b/modules/convert/package.go
new file mode 100644
index 0000000000..681219ca1a
--- /dev/null
+++ b/modules/convert/package.go
@@ -0,0 +1,43 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package convert
+
+import (
+ "code.gitea.io/gitea/models/packages"
+ "code.gitea.io/gitea/models/perm"
+ api "code.gitea.io/gitea/modules/structs"
+)
+
+// ToPackage convert a packages.PackageDescriptor to api.Package
+func ToPackage(pd *packages.PackageDescriptor) *api.Package {
+ var repo *api.Repository
+ if pd.Repository != nil {
+ repo = ToRepo(pd.Repository, perm.AccessModeNone)
+ }
+
+ return &api.Package{
+ ID: pd.Version.ID,
+ Owner: ToUser(pd.Owner, nil),
+ Repository: repo,
+ Creator: ToUser(pd.Creator, nil),
+ Type: string(pd.Package.Type),
+ Name: pd.Package.Name,
+ Version: pd.Version.Version,
+ CreatedAt: pd.Version.CreatedUnix.AsTime(),
+ }
+}
+
+// ToPackageFile converts packages.PackageFileDescriptor to api.PackageFile
+func ToPackageFile(pfd *packages.PackageFileDescriptor) *api.PackageFile {
+ return &api.PackageFile{
+ ID: pfd.File.ID,
+ Size: pfd.Blob.Size,
+ Name: pfd.File.Name,
+ HashMD5: pfd.Blob.HashMD5,
+ HashSHA1: pfd.Blob.HashSHA1,
+ HashSHA256: pfd.Blob.HashSHA256,
+ HashSHA512: pfd.Blob.HashSHA512,
+ }
+}
diff --git a/modules/notification/base/notifier.go b/modules/notification/base/notifier.go
index 8174741169..2b8be18ad3 100644
--- a/modules/notification/base/notifier.go
+++ b/modules/notification/base/notifier.go
@@ -6,6 +6,7 @@ package base
import (
"code.gitea.io/gitea/models"
+ packages_model "code.gitea.io/gitea/models/packages"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/repository"
@@ -54,4 +55,6 @@ type Notifier interface {
NotifySyncCreateRef(doer *user_model.User, repo *repo_model.Repository, refType, refFullName, refID string)
NotifySyncDeleteRef(doer *user_model.User, repo *repo_model.Repository, refType, refFullName string)
NotifyRepoPendingTransfer(doer, newOwner *user_model.User, repo *repo_model.Repository)
+ NotifyPackageCreate(doer *user_model.User, pd *packages_model.PackageDescriptor)
+ NotifyPackageDelete(doer *user_model.User, pd *packages_model.PackageDescriptor)
}
diff --git a/modules/notification/base/null.go b/modules/notification/base/null.go
index 2bfcaafda9..29b5f0c97e 100644
--- a/modules/notification/base/null.go
+++ b/modules/notification/base/null.go
@@ -6,6 +6,7 @@ package base
import (
"code.gitea.io/gitea/models"
+ packages_model "code.gitea.io/gitea/models/packages"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/repository"
@@ -173,3 +174,11 @@ func (*NullNotifier) NotifySyncDeleteRef(doer *user_model.User, repo *repo_model
// NotifyRepoPendingTransfer places a place holder function
func (*NullNotifier) NotifyRepoPendingTransfer(doer, newOwner *user_model.User, repo *repo_model.Repository) {
}
+
+// NotifyPackageCreate places a place holder function
+func (*NullNotifier) NotifyPackageCreate(doer *user_model.User, pd *packages_model.PackageDescriptor) {
+}
+
+// NotifyPackageDelete places a place holder function
+func (*NullNotifier) NotifyPackageDelete(doer *user_model.User, pd *packages_model.PackageDescriptor) {
+}
diff --git a/modules/notification/notification.go b/modules/notification/notification.go
index a31e3810e2..90ff87941f 100644
--- a/modules/notification/notification.go
+++ b/modules/notification/notification.go
@@ -6,6 +6,7 @@ package notification
import (
"code.gitea.io/gitea/models"
+ packages_model "code.gitea.io/gitea/models/packages"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/notification/action"
@@ -306,3 +307,17 @@ func NotifyRepoPendingTransfer(doer, newOwner *user_model.User, repo *repo_model
notifier.NotifyRepoPendingTransfer(doer, newOwner, repo)
}
}
+
+// NotifyPackageCreate notifies creation of a package to notifiers
+func NotifyPackageCreate(doer *user_model.User, pd *packages_model.PackageDescriptor) {
+ for _, notifier := range notifiers {
+ notifier.NotifyPackageCreate(doer, pd)
+ }
+}
+
+// NotifyPackageDelete notifies deletion of a package to notifiers
+func NotifyPackageDelete(doer *user_model.User, pd *packages_model.PackageDescriptor) {
+ for _, notifier := range notifiers {
+ notifier.NotifyPackageDelete(doer, pd)
+ }
+}
diff --git a/modules/notification/webhook/webhook.go b/modules/notification/webhook/webhook.go
index d4d5eea6cb..94d4d180be 100644
--- a/modules/notification/webhook/webhook.go
+++ b/modules/notification/webhook/webhook.go
@@ -8,6 +8,7 @@ import (
"fmt"
"code.gitea.io/gitea/models"
+ packages_model "code.gitea.io/gitea/models/packages"
"code.gitea.io/gitea/models/perm"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unit"
@@ -855,3 +856,33 @@ func (m *webhookNotifier) NotifySyncCreateRef(pusher *user_model.User, repo *rep
func (m *webhookNotifier) NotifySyncDeleteRef(pusher *user_model.User, repo *repo_model.Repository, refType, refFullName string) {
m.NotifyDeleteRef(pusher, repo, refType, refFullName)
}
+
+func (m *webhookNotifier) NotifyPackageCreate(doer *user_model.User, pd *packages_model.PackageDescriptor) {
+ notifyPackage(doer, pd, api.HookPackageCreated)
+}
+
+func (m *webhookNotifier) NotifyPackageDelete(doer *user_model.User, pd *packages_model.PackageDescriptor) {
+ notifyPackage(doer, pd, api.HookPackageDeleted)
+}
+
+func notifyPackage(sender *user_model.User, pd *packages_model.PackageDescriptor, action api.HookPackageAction) {
+ if pd.Repository == nil {
+ // TODO https://github.com/go-gitea/gitea/pull/17940
+ return
+ }
+
+ org := pd.Owner
+ if !org.IsOrganization() {
+ org = nil
+ }
+
+ if err := webhook_services.PrepareWebhooks(pd.Repository, webhook.HookEventPackage, &api.PackagePayload{
+ Action: action,
+ Repository: convert.ToRepo(pd.Repository, perm.AccessModeNone),
+ Package: convert.ToPackage(pd),
+ Organization: convert.ToUser(org, nil),
+ Sender: convert.ToUser(sender, nil),
+ }); err != nil {
+ log.Error("PrepareWebhooks: %v", err)
+ }
+}
diff --git a/modules/packages/composer/metadata.go b/modules/packages/composer/metadata.go
new file mode 100644
index 0000000000..797576b1e7
--- /dev/null
+++ b/modules/packages/composer/metadata.go
@@ -0,0 +1,147 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package composer
+
+import (
+ "archive/zip"
+ "errors"
+ "io"
+ "regexp"
+ "strings"
+
+ "code.gitea.io/gitea/modules/json"
+ "code.gitea.io/gitea/modules/validation"
+
+ "github.com/hashicorp/go-version"
+)
+
+// TypeProperty is the name of the property for Composer package types
+const TypeProperty = "composer.type"
+
+var (
+ // ErrMissingComposerFile indicates a missing composer.json file
+ ErrMissingComposerFile = errors.New("composer.json file is missing")
+ // ErrInvalidName indicates an invalid package name
+ ErrInvalidName = errors.New("package name is invalid")
+ // ErrInvalidVersion indicates an invalid package version
+ ErrInvalidVersion = errors.New("package version is invalid")
+)
+
+// Package represents a Composer package
+type Package struct {
+ Name string
+ Version string
+ Type string
+ Metadata *Metadata
+}
+
+// Metadata represents the metadata of a Composer package
+type Metadata struct {
+ Description string `json:"description,omitempty"`
+ Keywords []string `json:"keywords,omitempty"`
+ Homepage string `json:"homepage,omitempty"`
+ License Licenses `json:"license,omitempty"`
+ Authors []Author `json:"authors,omitempty"`
+ Autoload map[string]interface{} `json:"autoload,omitempty"`
+ AutoloadDev map[string]interface{} `json:"autoload-dev,omitempty"`
+ Extra map[string]interface{} `json:"extra,omitempty"`
+ Require map[string]string `json:"require,omitempty"`
+ RequireDev map[string]string `json:"require-dev,omitempty"`
+ Suggest map[string]string `json:"suggest,omitempty"`
+ Provide map[string]string `json:"provide,omitempty"`
+}
+
+// Licenses represents the licenses of a Composer package
+type Licenses []string
+
+// UnmarshalJSON reads from a string or array
+func (l *Licenses) UnmarshalJSON(data []byte) error {
+ switch data[0] {
+ case '"':
+ var value string
+ if err := json.Unmarshal(data, &value); err != nil {
+ return err
+ }
+ *l = Licenses{value}
+ case '[':
+ values := make([]string, 0, 5)
+ if err := json.Unmarshal(data, &values); err != nil {
+ return err
+ }
+ *l = Licenses(values)
+ }
+ return nil
+}
+
+// Author represents an author
+type Author struct {
+ Name string `json:"name,omitempty"`
+ Email string `json:"email,omitempty"`
+ Homepage string `json:"homepage,omitempty"`
+}
+
+var nameMatch = regexp.MustCompile(`\A[a-z0-9]([_\.-]?[a-z0-9]+)*/[a-z0-9](([_\.]?|-{0,2})[a-z0-9]+)*\z`)
+
+// ParsePackage parses the metadata of a Composer package file
+func ParsePackage(r io.ReaderAt, size int64) (*Package, error) {
+ archive, err := zip.NewReader(r, size)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, file := range archive.File {
+ if strings.Count(file.Name, "/") > 1 {
+ continue
+ }
+ if strings.HasSuffix(strings.ToLower(file.Name), "composer.json") {
+ f, err := archive.Open(file.Name)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ return ParseComposerFile(f)
+ }
+ }
+ return nil, ErrMissingComposerFile
+}
+
+// ParseComposerFile parses a composer.json file to retrieve the metadata of a Composer package
+func ParseComposerFile(r io.Reader) (*Package, error) {
+ var cj struct {
+ Name string `json:"name"`
+ Version string `json:"version"`
+ Type string `json:"type"`
+ Metadata
+ }
+ if err := json.NewDecoder(r).Decode(&cj); err != nil {
+ return nil, err
+ }
+
+ if !nameMatch.MatchString(cj.Name) {
+ return nil, ErrInvalidName
+ }
+
+ if cj.Version != "" {
+ if _, err := version.NewSemver(cj.Version); err != nil {
+ return nil, ErrInvalidVersion
+ }
+ }
+
+ if !validation.IsValidURL(cj.Homepage) {
+ cj.Homepage = ""
+ }
+
+ if cj.Type == "" {
+ cj.Type = "library"
+ }
+
+ return &Package{
+ Name: cj.Name,
+ Version: cj.Version,
+ Type: cj.Type,
+ Metadata: &cj.Metadata,
+ }, nil
+}
diff --git a/modules/packages/composer/metadata_test.go b/modules/packages/composer/metadata_test.go
new file mode 100644
index 0000000000..feadc18b6a
--- /dev/null
+++ b/modules/packages/composer/metadata_test.go
@@ -0,0 +1,130 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package composer
+
+import (
+ "archive/zip"
+ "bytes"
+ "strings"
+ "testing"
+
+ "code.gitea.io/gitea/modules/json"
+
+ "github.com/stretchr/testify/assert"
+)
+
+const (
+ name = "gitea/composer-package"
+ description = "Package Description"
+ packageType = "composer-plugin"
+ author = "Gitea Authors"
+ email = "no.reply@gitea.io"
+ homepage = "https://gitea.io"
+ license = "MIT"
+)
+
+const composerContent = `{
+ "name": "` + name + `",
+ "description": "` + description + `",
+ "type": "` + packageType + `",
+ "license": "` + license + `",
+ "authors": [
+ {
+ "name": "` + author + `",
+ "email": "` + email + `"
+ }
+ ],
+ "homepage": "` + homepage + `",
+ "autoload": {
+ "psr-4": {"Gitea\\ComposerPackage\\": "src/"}
+ },
+ "require": {
+ "php": ">=7.2 || ^8.0"
+ }
+}`
+
+func TestLicenseUnmarshal(t *testing.T) {
+ var l Licenses
+ assert.NoError(t, json.NewDecoder(strings.NewReader(`["MIT"]`)).Decode(&l))
+ assert.Len(t, l, 1)
+ assert.Equal(t, "MIT", l[0])
+ assert.NoError(t, json.NewDecoder(strings.NewReader(`"MIT"`)).Decode(&l))
+ assert.Len(t, l, 1)
+ assert.Equal(t, "MIT", l[0])
+}
+
+func TestParsePackage(t *testing.T) {
+ createArchive := func(name, content string) []byte {
+ var buf bytes.Buffer
+ archive := zip.NewWriter(&buf)
+ w, _ := archive.Create(name)
+ w.Write([]byte(content))
+ archive.Close()
+ return buf.Bytes()
+ }
+
+ t.Run("MissingComposerFile", func(t *testing.T) {
+ data := createArchive("dummy.txt", "")
+
+ cp, err := ParsePackage(bytes.NewReader(data), int64(len(data)))
+ assert.Nil(t, cp)
+ assert.ErrorIs(t, err, ErrMissingComposerFile)
+ })
+
+ t.Run("MissingComposerFileInRoot", func(t *testing.T) {
+ data := createArchive("sub/sub/composer.json", "")
+
+ cp, err := ParsePackage(bytes.NewReader(data), int64(len(data)))
+ assert.Nil(t, cp)
+ assert.ErrorIs(t, err, ErrMissingComposerFile)
+ })
+
+ t.Run("InvalidComposerFile", func(t *testing.T) {
+ data := createArchive("composer.json", "")
+
+ cp, err := ParsePackage(bytes.NewReader(data), int64(len(data)))
+ assert.Nil(t, cp)
+ assert.Error(t, err)
+ })
+
+ t.Run("Valid", func(t *testing.T) {
+ data := createArchive("composer.json", composerContent)
+
+ cp, err := ParsePackage(bytes.NewReader(data), int64(len(data)))
+ assert.NoError(t, err)
+ assert.NotNil(t, cp)
+ })
+}
+
+func TestParseComposerFile(t *testing.T) {
+ t.Run("InvalidPackageName", func(t *testing.T) {
+ cp, err := ParseComposerFile(strings.NewReader(`{}`))
+ assert.Nil(t, cp)
+ assert.ErrorIs(t, err, ErrInvalidName)
+ })
+
+ t.Run("InvalidPackageVersion", func(t *testing.T) {
+ cp, err := ParseComposerFile(strings.NewReader(`{"name": "gitea/composer-package", "version": "1.a.3"}`))
+ assert.Nil(t, cp)
+ assert.ErrorIs(t, err, ErrInvalidVersion)
+ })
+
+ t.Run("Valid", func(t *testing.T) {
+ cp, err := ParseComposerFile(strings.NewReader(composerContent))
+ assert.NoError(t, err)
+ assert.NotNil(t, cp)
+
+ assert.Equal(t, name, cp.Name)
+ assert.Empty(t, cp.Version)
+ assert.Equal(t, description, cp.Metadata.Description)
+ assert.Len(t, cp.Metadata.Authors, 1)
+ assert.Equal(t, author, cp.Metadata.Authors[0].Name)
+ assert.Equal(t, email, cp.Metadata.Authors[0].Email)
+ assert.Equal(t, homepage, cp.Metadata.Homepage)
+ assert.Equal(t, packageType, cp.Type)
+ assert.Len(t, cp.Metadata.License, 1)
+ assert.Equal(t, license, cp.Metadata.License[0])
+ })
+}
diff --git a/modules/packages/conan/conanfile_parser.go b/modules/packages/conan/conanfile_parser.go
new file mode 100644
index 0000000000..960e813533
--- /dev/null
+++ b/modules/packages/conan/conanfile_parser.go
@@ -0,0 +1,68 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package conan
+
+import (
+ "io"
+ "regexp"
+ "strings"
+)
+
+var (
+ patternAuthor = compilePattern("author")
+ patternHomepage = compilePattern("homepage")
+ patternURL = compilePattern("url")
+ patternLicense = compilePattern("license")
+ patternDescription = compilePattern("description")
+ patternTopics = regexp.MustCompile(`(?im)^\s*topics\s*=\s*\((.+)\)`)
+ patternTopicList = regexp.MustCompile(`\s*['"](.+?)['"]\s*,?`)
+)
+
+func compilePattern(name string) *regexp.Regexp {
+ return regexp.MustCompile(`(?im)^\s*` + name + `\s*=\s*['"\(](.+)['"\)]`)
+}
+
+func ParseConanfile(r io.Reader) (*Metadata, error) {
+ buf, err := io.ReadAll(io.LimitReader(r, 1<<20))
+ if err != nil {
+ return nil, err
+ }
+
+ metadata := &Metadata{}
+
+ m := patternAuthor.FindSubmatch(buf)
+ if len(m) > 1 && len(m[1]) > 0 {
+ metadata.Author = string(m[1])
+ }
+ m = patternHomepage.FindSubmatch(buf)
+ if len(m) > 1 && len(m[1]) > 0 {
+ metadata.ProjectURL = string(m[1])
+ }
+ m = patternURL.FindSubmatch(buf)
+ if len(m) > 1 && len(m[1]) > 0 {
+ metadata.RepositoryURL = string(m[1])
+ }
+ m = patternLicense.FindSubmatch(buf)
+ if len(m) > 1 && len(m[1]) > 0 {
+ metadata.License = strings.ReplaceAll(strings.ReplaceAll(string(m[1]), "'", ""), "\"", "")
+ }
+ m = patternDescription.FindSubmatch(buf)
+ if len(m) > 1 && len(m[1]) > 0 {
+ metadata.Description = string(m[1])
+ }
+ m = patternTopics.FindSubmatch(buf)
+ if len(m) > 1 && len(m[1]) > 0 {
+ m2 := patternTopicList.FindAllSubmatch(m[1], -1)
+ if len(m2) > 0 {
+ metadata.Keywords = make([]string, 0, len(m2))
+ for _, g := range m2 {
+ if len(g) > 1 {
+ metadata.Keywords = append(metadata.Keywords, string(g[1]))
+ }
+ }
+ }
+ }
+ return metadata, nil
+}
diff --git a/modules/packages/conan/conanfile_parser_test.go b/modules/packages/conan/conanfile_parser_test.go
new file mode 100644
index 0000000000..0ac9c87b14
--- /dev/null
+++ b/modules/packages/conan/conanfile_parser_test.go
@@ -0,0 +1,51 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package conan
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+const (
+ name = "ConanPackage"
+ version = "1.2"
+ license = "MIT"
+ author = "Gitea <info@gitea.io>"
+ homepage = "https://gitea.io/"
+ url = "https://gitea.com/"
+ description = "Description of ConanPackage"
+ topic1 = "gitea"
+ topic2 = "conan"
+ contentConanfile = `from conans import ConanFile, CMake, tools
+
+class ConanPackageConan(ConanFile):
+ name = "` + name + `"
+ version = "` + version + `"
+ license = "` + license + `"
+ author = "` + author + `"
+ homepage = "` + homepage + `"
+ url = "` + url + `"
+ description = "` + description + `"
+ topics = ("` + topic1 + `", "` + topic2 + `")
+ settings = "os", "compiler", "build_type", "arch"
+ options = {"shared": [True, False], "fPIC": [True, False]}
+ default_options = {"shared": False, "fPIC": True}
+ generators = "cmake"
+`
+)
+
+func TestParseConanfile(t *testing.T) {
+ metadata, err := ParseConanfile(strings.NewReader(contentConanfile))
+ assert.Nil(t, err)
+ assert.Equal(t, license, metadata.License)
+ assert.Equal(t, author, metadata.Author)
+ assert.Equal(t, homepage, metadata.ProjectURL)
+ assert.Equal(t, url, metadata.RepositoryURL)
+ assert.Equal(t, description, metadata.Description)
+ assert.Equal(t, []string{topic1, topic2}, metadata.Keywords)
+}
diff --git a/modules/packages/conan/conaninfo_parser.go b/modules/packages/conan/conaninfo_parser.go
new file mode 100644
index 0000000000..bb228e0207
--- /dev/null
+++ b/modules/packages/conan/conaninfo_parser.go
@@ -0,0 +1,123 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package conan
+
+import (
+ "bufio"
+ "errors"
+ "io"
+ "strings"
+)
+
+// Conaninfo represents infos of a Conan package
+type Conaninfo struct {
+ Settings map[string]string `json:"settings"`
+ FullSettings map[string]string `json:"full_settings"`
+ Requires []string `json:"requires"`
+ FullRequires []string `json:"full_requires"`
+ Options map[string]string `json:"options"`
+ FullOptions map[string]string `json:"full_options"`
+ RecipeHash string `json:"recipe_hash"`
+ Environment map[string][]string `json:"environment"`
+}
+
+func ParseConaninfo(r io.Reader) (*Conaninfo, error) {
+ sections, err := readSections(io.LimitReader(r, 1<<20))
+ if err != nil {
+ return nil, err
+ }
+
+ info := &Conaninfo{}
+ for section, lines := range sections {
+ if len(lines) == 0 {
+ continue
+ }
+ switch section {
+ case "settings":
+ info.Settings = toMap(lines)
+ case "full_settings":
+ info.FullSettings = toMap(lines)
+ case "options":
+ info.Options = toMap(lines)
+ case "full_options":
+ info.FullOptions = toMap(lines)
+ case "requires":
+ info.Requires = lines
+ case "full_requires":
+ info.FullRequires = lines
+ case "recipe_hash":
+ info.RecipeHash = lines[0]
+ case "env":
+ info.Environment = toMapArray(lines)
+ }
+ }
+ return info, nil
+}
+
+func readSections(r io.Reader) (map[string][]string, error) {
+ sections := make(map[string][]string)
+
+ section := ""
+ lines := make([]string, 0, 5)
+
+ scanner := bufio.NewScanner(r)
+ for scanner.Scan() {
+ line := strings.TrimSpace(scanner.Text())
+ if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") {
+ if section != "" {
+ sections[section] = lines
+ }
+ section = line[1 : len(line)-1]
+ lines = make([]string, 0, 5)
+ continue
+ }
+ if section != "" {
+ if line != "" {
+ lines = append(lines, line)
+ }
+ continue
+ }
+ if line != "" {
+ return nil, errors.New("Invalid conaninfo.txt")
+ }
+ }
+ if err := scanner.Err(); err != nil {
+ return nil, err
+ }
+ if section != "" {
+ sections[section] = lines
+ }
+ return sections, nil
+}
+
+func toMap(lines []string) map[string]string {
+ result := make(map[string]string)
+ for _, line := range lines {
+ parts := strings.SplitN(line, "=", 2)
+ if len(parts) != 2 || len(parts[0]) == 0 || len(parts[1]) == 0 {
+ continue
+ }
+ result[parts[0]] = parts[1]
+ }
+ return result
+}
+
+func toMapArray(lines []string) map[string][]string {
+ result := make(map[string][]string)
+ for _, line := range lines {
+ parts := strings.SplitN(line, "=", 2)
+ if len(parts) != 2 || len(parts[0]) == 0 || len(parts[1]) == 0 {
+ continue
+ }
+ var items []string
+ if strings.HasPrefix(parts[1], "[") && strings.HasSuffix(parts[1], "]") {
+ items = strings.Split(parts[1], ",")
+ } else {
+ items = []string{parts[1]}
+ }
+ result[parts[0]] = items
+ }
+ return result
+}
diff --git a/modules/packages/conan/conaninfo_parser_test.go b/modules/packages/conan/conaninfo_parser_test.go
new file mode 100644
index 0000000000..3e28191b06
--- /dev/null
+++ b/modules/packages/conan/conaninfo_parser_test.go
@@ -0,0 +1,85 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package conan
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+const (
+ settingsKey = "arch"
+ settingsValue = "x84_64"
+ optionsKey = "shared"
+ optionsValue = "False"
+ requires = "fmt/7.1.3"
+ hash = "74714915a51073acb548ca1ce29afbac"
+ envKey = "CC"
+ envValue = "gcc-10"
+
+ contentConaninfo = `[settings]
+ ` + settingsKey + `=` + settingsValue + `
+
+[requires]
+ ` + requires + `
+
+[options]
+ ` + optionsKey + `=` + optionsValue + `
+
+[full_settings]
+ ` + settingsKey + `=` + settingsValue + `
+
+[full_requires]
+ ` + requires + `
+
+[full_options]
+ ` + optionsKey + `=` + optionsValue + `
+
+[recipe_hash]
+ ` + hash + `
+
+[env]
+` + envKey + `=` + envValue + `
+
+`
+)
+
+func TestParseConaninfo(t *testing.T) {
+ info, err := ParseConaninfo(strings.NewReader(contentConaninfo))
+ assert.NotNil(t, info)
+ assert.Nil(t, err)
+ assert.Equal(
+ t,
+ map[string]string{
+ settingsKey: settingsValue,
+ },
+ info.Settings,
+ )
+ assert.Equal(t, info.Settings, info.FullSettings)
+ assert.Equal(
+ t,
+ map[string]string{
+ optionsKey: optionsValue,
+ },
+ info.Options,
+ )
+ assert.Equal(t, info.Options, info.FullOptions)
+ assert.Equal(
+ t,
+ []string{requires},
+ info.Requires,
+ )
+ assert.Equal(t, info.Requires, info.FullRequires)
+ assert.Equal(t, hash, info.RecipeHash)
+ assert.Equal(
+ t,
+ map[string][]string{
+ envKey: {envValue},
+ },
+ info.Environment,
+ )
+}
diff --git a/modules/packages/conan/metadata.go b/modules/packages/conan/metadata.go
new file mode 100644
index 0000000000..a7d6a9df0b
--- /dev/null
+++ b/modules/packages/conan/metadata.go
@@ -0,0 +1,24 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package conan
+
+const (
+ PropertyRecipeUser = "conan.recipe.user"
+ PropertyRecipeChannel = "conan.recipe.channel"
+ PropertyRecipeRevision = "conan.recipe.revision"
+ PropertyPackageReference = "conan.package.reference"
+ PropertyPackageRevision = "conan.package.revision"
+ PropertyPackageInfo = "conan.package.info"
+)
+
+// Metadata represents the metadata of a Conan package
+type Metadata struct {
+ Author string `json:"author,omitempty"`
+ License string `json:"license,omitempty"`
+ ProjectURL string `json:"project_url,omitempty"`
+ RepositoryURL string `json:"repository_url,omitempty"`
+ Description string `json:"description,omitempty"`
+ Keywords []string `json:"keywords,omitempty"`
+}
diff --git a/modules/packages/conan/reference.go b/modules/packages/conan/reference.go
new file mode 100644
index 0000000000..c43446e6e5
--- /dev/null
+++ b/modules/packages/conan/reference.go
@@ -0,0 +1,155 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package conan
+
+import (
+ "errors"
+ "fmt"
+ "regexp"
+
+ "code.gitea.io/gitea/modules/log"
+
+ goversion "github.com/hashicorp/go-version"
+)
+
+const (
+ // taken from https://github.com/conan-io/conan/blob/develop/conans/model/ref.py
+ minChars = 2
+ maxChars = 51
+
+ // DefaultRevision if no revision is specified
+ DefaultRevision = "0"
+)
+
+var (
+ namePattern = regexp.MustCompile(fmt.Sprintf(`^[a-zA-Z0-9_][a-zA-Z0-9_\+\.-]{%d,%d}$`, minChars-1, maxChars-1))
+ revisionPattern = regexp.MustCompile(fmt.Sprintf(`^[a-zA-Z0-9]{1,%d}$`, maxChars))
+
+ ErrValidation = errors.New("Could not validate one or more reference fields")
+)
+
+// RecipeReference represents a recipe <Name>/<Version>@<User>/<Channel>#<Revision>
+type RecipeReference struct {
+ Name string
+ Version string
+ User string
+ Channel string
+ Revision string
+}
+
+func NewRecipeReference(name, version, user, channel, revision string) (*RecipeReference, error) {
+ log.Trace("Conan Recipe: %s/%s(@%s/%s(#%s))", name, version, user, channel, revision)
+
+ if user == "_" {
+ user = ""
+ }
+ if channel == "_" {
+ channel = ""
+ }
+
+ if (user != "" && channel == "") || (user == "" && channel != "") {
+ return nil, ErrValidation
+ }
+
+ if !namePattern.MatchString(name) {
+ return nil, ErrValidation
+ }
+ if _, err := goversion.NewSemver(version); err != nil {
+ return nil, ErrValidation
+ }
+ if user != "" && !namePattern.MatchString(user) {
+ return nil, ErrValidation
+ }
+ if channel != "" && !namePattern.MatchString(channel) {
+ return nil, ErrValidation
+ }
+ if revision != "" && !revisionPattern.MatchString(revision) {
+ return nil, ErrValidation
+ }
+
+ return &RecipeReference{name, version, user, channel, revision}, nil
+}
+
+func (r *RecipeReference) RevisionOrDefault() string {
+ if r.Revision == "" {
+ return DefaultRevision
+ }
+ return r.Revision
+}
+
+func (r *RecipeReference) String() string {
+ rev := ""
+ if r.Revision != "" {
+ rev = "#" + r.Revision
+ }
+ if r.User == "" || r.Channel == "" {
+ return fmt.Sprintf("%s/%s%s", r.Name, r.Version, rev)
+ }
+ return fmt.Sprintf("%s/%s@%s/%s%s", r.Name, r.Version, r.User, r.Channel, rev)
+}
+
+func (r *RecipeReference) LinkName() string {
+ user := r.User
+ if user == "" {
+ user = "_"
+ }
+ channel := r.Channel
+ if channel == "" {
+ channel = "_"
+ }
+ return fmt.Sprintf("%s/%s/%s/%s/%s", r.Name, r.Version, user, channel, r.RevisionOrDefault())
+}
+
+func (r *RecipeReference) WithRevision(revision string) *RecipeReference {
+ return &RecipeReference{r.Name, r.Version, r.User, r.Channel, revision}
+}
+
+// AsKey builds the additional key for the package file
+func (r *RecipeReference) AsKey() string {
+ return fmt.Sprintf("%s|%s|%s", r.User, r.Channel, r.RevisionOrDefault())
+}
+
+// PackageReference represents a package of a recipe <Name>/<Version>@<User>/<Channel>#<Revision> <Reference>#<Revision>
+type PackageReference struct {
+ Recipe *RecipeReference
+ Reference string
+ Revision string
+}
+
+func NewPackageReference(recipe *RecipeReference, reference, revision string) (*PackageReference, error) {
+ log.Trace("Conan Package: %v %s(#%s)", recipe, reference, revision)
+
+ if recipe == nil {
+ return nil, ErrValidation
+ }
+ if reference == "" || !revisionPattern.MatchString(reference) {
+ return nil, ErrValidation
+ }
+ if revision != "" && !revisionPattern.MatchString(revision) {
+ return nil, ErrValidation
+ }
+
+ return &PackageReference{recipe, reference, revision}, nil
+}
+
+func (r *PackageReference) RevisionOrDefault() string {
+ if r.Revision == "" {
+ return DefaultRevision
+ }
+ return r.Revision
+}
+
+func (r *PackageReference) LinkName() string {
+ return fmt.Sprintf("%s/%s", r.Reference, r.RevisionOrDefault())
+}
+
+func (r *PackageReference) WithRevision(revision string) *PackageReference {
+ return &PackageReference{r.Recipe, r.Reference, revision}
+}
+
+// AsKey builds the additional key for the package file
+func (r *PackageReference) AsKey() string {
+ return fmt.Sprintf("%s|%s|%s|%s|%s", r.Recipe.User, r.Recipe.Channel, r.Recipe.RevisionOrDefault(), r.Reference, r.RevisionOrDefault())
+}
diff --git a/modules/packages/conan/reference_test.go b/modules/packages/conan/reference_test.go
new file mode 100644
index 0000000000..29ba3a543b
--- /dev/null
+++ b/modules/packages/conan/reference_test.go
@@ -0,0 +1,147 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package conan
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestNewRecipeReference(t *testing.T) {
+ cases := []struct {
+ Name string
+ Version string
+ User string
+ Channel string
+ Revision string
+ IsValid bool
+ }{
+ {"", "", "", "", "", false},
+ {"name", "", "", "", "", false},
+ {"", "1.0", "", "", "", false},
+ {"", "", "user", "", "", false},
+ {"", "", "", "channel", "", false},
+ {"", "", "", "", "0", false},
+ {"name", "1.0", "", "", "", true},
+ {"name", "1.0", "user", "", "", false},
+ {"name", "1.0", "", "channel", "", false},
+ {"name", "1.0", "user", "channel", "", true},
+ {"name", "1.0", "_", "", "", true},
+ {"name", "1.0", "", "_", "", true},
+ {"name", "1.0", "_", "_", "", true},
+ {"name", "1.0", "_", "_", "0", true},
+ {"name", "1.0", "", "", "0", true},
+ {"name", "1.0", "", "", "000000000000000000000000000000000000000000000000000000000000", false},
+ }
+
+ for i, c := range cases {
+ rref, err := NewRecipeReference(c.Name, c.Version, c.User, c.Channel, c.Revision)
+ if c.IsValid {
+ assert.NoError(t, err, "case %d, should be invalid", i)
+ assert.NotNil(t, rref, "case %d, should not be nil", i)
+ } else {
+ assert.Error(t, err, "case %d, should be valid", i)
+ }
+ }
+}
+
+func TestRecipeReferenceRevisionOrDefault(t *testing.T) {
+ rref, err := NewRecipeReference("name", "1.0", "", "", "")
+ assert.NoError(t, err)
+ assert.Equal(t, DefaultRevision, rref.RevisionOrDefault())
+
+ rref, err = NewRecipeReference("name", "1.0", "", "", DefaultRevision)
+ assert.NoError(t, err)
+ assert.Equal(t, DefaultRevision, rref.RevisionOrDefault())
+
+ rref, err = NewRecipeReference("name", "1.0", "", "", "Az09")
+ assert.NoError(t, err)
+ assert.Equal(t, "Az09", rref.RevisionOrDefault())
+}
+
+func TestRecipeReferenceString(t *testing.T) {
+ rref, err := NewRecipeReference("name", "1.0", "", "", "")
+ assert.NoError(t, err)
+ assert.Equal(t, "name/1.0", rref.String())
+
+ rref, err = NewRecipeReference("name", "1.0", "user", "channel", "")
+ assert.NoError(t, err)
+ assert.Equal(t, "name/1.0@user/channel", rref.String())
+
+ rref, err = NewRecipeReference("name", "1.0", "user", "channel", "Az09")
+ assert.NoError(t, err)
+ assert.Equal(t, "name/1.0@user/channel#Az09", rref.String())
+}
+
+func TestRecipeReferenceLinkName(t *testing.T) {
+ rref, err := NewRecipeReference("name", "1.0", "", "", "")
+ assert.NoError(t, err)
+ assert.Equal(t, "name/1.0/_/_/0", rref.LinkName())
+
+ rref, err = NewRecipeReference("name", "1.0", "user", "channel", "")
+ assert.NoError(t, err)
+ assert.Equal(t, "name/1.0/user/channel/0", rref.LinkName())
+
+ rref, err = NewRecipeReference("name", "1.0", "user", "channel", "Az09")
+ assert.NoError(t, err)
+ assert.Equal(t, "name/1.0/user/channel/Az09", rref.LinkName())
+}
+
+func TestNewPackageReference(t *testing.T) {
+ rref, _ := NewRecipeReference("name", "1.0", "", "", "")
+
+ cases := []struct {
+ Recipe *RecipeReference
+ Reference string
+ Revision string
+ IsValid bool
+ }{
+ {nil, "", "", false},
+ {rref, "", "", false},
+ {nil, "aZ09", "", false},
+ {rref, "aZ09", "", true},
+ {rref, "", "Az09", false},
+ {rref, "aZ09", "Az09", true},
+ }
+
+ for i, c := range cases {
+ pref, err := NewPackageReference(c.Recipe, c.Reference, c.Revision)
+ if c.IsValid {
+ assert.NoError(t, err, "case %d, should be invalid", i)
+ assert.NotNil(t, pref, "case %d, should not be nil", i)
+ } else {
+ assert.Error(t, err, "case %d, should be valid", i)
+ }
+ }
+}
+
+func TestPackageReferenceRevisionOrDefault(t *testing.T) {
+ rref, _ := NewRecipeReference("name", "1.0", "", "", "")
+
+ pref, err := NewPackageReference(rref, "ref", "")
+ assert.NoError(t, err)
+ assert.Equal(t, DefaultRevision, pref.RevisionOrDefault())
+
+ pref, err = NewPackageReference(rref, "ref", DefaultRevision)
+ assert.NoError(t, err)
+ assert.Equal(t, DefaultRevision, pref.RevisionOrDefault())
+
+ pref, err = NewPackageReference(rref, "ref", "Az09")
+ assert.NoError(t, err)
+ assert.Equal(t, "Az09", pref.RevisionOrDefault())
+}
+
+func TestPackageReferenceLinkName(t *testing.T) {
+ rref, _ := NewRecipeReference("name", "1.0", "", "", "")
+
+ pref, err := NewPackageReference(rref, "ref", "")
+ assert.NoError(t, err)
+ assert.Equal(t, "ref/0", pref.LinkName())
+
+ pref, err = NewPackageReference(rref, "ref", "Az09")
+ assert.NoError(t, err)
+ assert.Equal(t, "ref/Az09", pref.LinkName())
+}
diff --git a/modules/packages/container/helm/helm.go b/modules/packages/container/helm/helm.go
new file mode 100644
index 0000000000..98d3824a85
--- /dev/null
+++ b/modules/packages/container/helm/helm.go
@@ -0,0 +1,56 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package helm
+
+// https://github.com/helm/helm/blob/main/pkg/chart/
+
+const ConfigMediaType = "application/vnd.cncf.helm.config.v1+json"
+
+// Maintainer describes a Chart maintainer.
+type Maintainer struct {
+ // Name is a user name or organization name
+ Name string `json:"name,omitempty"`
+ // Email is an optional email address to contact the named maintainer
+ Email string `json:"email,omitempty"`
+ // URL is an optional URL to an address for the named maintainer
+ URL string `json:"url,omitempty"`
+}
+
+// Metadata for a Chart file. This models the structure of a Chart.yaml file.
+type Metadata struct {
+ // The name of the chart. Required.
+ Name string `json:"name,omitempty"`
+ // The URL to a relevant project page, git repo, or contact person
+ Home string `json:"home,omitempty"`
+ // Source is the URL to the source code of this chart
+ Sources []string `json:"sources,omitempty"`
+ // A SemVer 2 conformant version string of the chart. Required.
+ Version string `json:"version,omitempty"`
+ // A one-sentence description of the chart
+ Description string `json:"description,omitempty"`
+ // A list of string keywords
+ Keywords []string `json:"keywords,omitempty"`
+ // A list of name and URL/email address combinations for the maintainer(s)
+ Maintainers []*Maintainer `json:"maintainers,omitempty"`
+ // The URL to an icon file.
+ Icon string `json:"icon,omitempty"`
+ // The API Version of this chart. Required.
+ APIVersion string `json:"apiVersion,omitempty"`
+ // The condition to check to enable chart
+ Condition string `json:"condition,omitempty"`
+ // The tags to check to enable chart
+ Tags string `json:"tags,omitempty"`
+ // The version of the application enclosed inside of this chart.
+ AppVersion string `json:"appVersion,omitempty"`
+ // Whether or not this chart is deprecated
+ Deprecated bool `json:"deprecated,omitempty"`
+ // Annotations are additional mappings uninterpreted by Helm,
+ // made available for inspection by other applications.
+ Annotations map[string]string `json:"annotations,omitempty"`
+ // KubeVersion is a SemVer constraint specifying the version of Kubernetes required.
+ KubeVersion string `json:"kubeVersion,omitempty"`
+ // Specifies the chart type: application or library
+ Type string `json:"type,omitempty"`
+}
diff --git a/modules/packages/container/metadata.go b/modules/packages/container/metadata.go
new file mode 100644
index 0000000000..087d38e5bd
--- /dev/null
+++ b/modules/packages/container/metadata.go
@@ -0,0 +1,157 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package container
+
+import (
+ "fmt"
+ "io"
+ "strings"
+
+ "code.gitea.io/gitea/modules/json"
+ "code.gitea.io/gitea/modules/packages/container/helm"
+ "code.gitea.io/gitea/modules/packages/container/oci"
+ "code.gitea.io/gitea/modules/validation"
+)
+
+const (
+ PropertyDigest = "container.digest"
+ PropertyMediaType = "container.mediatype"
+ PropertyManifestTagged = "container.manifest.tagged"
+ PropertyManifestReference = "container.manifest.reference"
+
+ DefaultPlatform = "linux/amd64"
+
+ labelLicenses = "org.opencontainers.image.licenses"
+ labelURL = "org.opencontainers.image.url"
+ labelSource = "org.opencontainers.image.source"
+ labelDocumentation = "org.opencontainers.image.documentation"
+ labelDescription = "org.opencontainers.image.description"
+ labelAuthors = "org.opencontainers.image.authors"
+)
+
+type ImageType string
+
+const (
+ TypeOCI ImageType = "oci"
+ TypeHelm ImageType = "helm"
+)
+
+// Name gets the name of the image type
+func (it ImageType) Name() string {
+ switch it {
+ case TypeHelm:
+ return "Helm Chart"
+ default:
+ return "OCI / Docker"
+ }
+}
+
+// Metadata represents the metadata of a Container package
+type Metadata struct {
+ Type ImageType `json:"type"`
+ IsTagged bool `json:"is_tagged"`
+ Platform string `json:"platform,omitempty"`
+ Description string `json:"description,omitempty"`
+ Authors []string `json:"authors,omitempty"`
+ Licenses string `json:"license,omitempty"`
+ ProjectURL string `json:"project_url,omitempty"`
+ RepositoryURL string `json:"repository_url,omitempty"`
+ DocumentationURL string `json:"documentation_url,omitempty"`
+ Labels map[string]string `json:"labels,omitempty"`
+ ImageLayers []string `json:"layer_creation,omitempty"`
+ MultiArch map[string]string `json:"multiarch,omitempty"`
+}
+
+// ParseImageConfig parses the metadata of an image config
+func ParseImageConfig(mediaType oci.MediaType, r io.Reader) (*Metadata, error) {
+ if strings.EqualFold(string(mediaType), helm.ConfigMediaType) {
+ return parseHelmConfig(r)
+ }
+
+ // fallback to OCI Image Config
+ return parseOCIImageConfig(r)
+}
+
+func parseOCIImageConfig(r io.Reader) (*Metadata, error) {
+ var image oci.Image
+ if err := json.NewDecoder(r).Decode(&image); err != nil {
+ return nil, err
+ }
+
+ platform := DefaultPlatform
+ if image.OS != "" && image.Architecture != "" {
+ platform = fmt.Sprintf("%s/%s", image.OS, image.Architecture)
+ if image.Variant != "" {
+ platform = fmt.Sprintf("%s/%s", platform, image.Variant)
+ }
+ }
+
+ imageLayers := make([]string, 0, len(image.History))
+ for _, history := range image.History {
+ cmd := history.CreatedBy
+ if i := strings.Index(cmd, "#(nop) "); i != -1 {
+ cmd = strings.TrimSpace(cmd[i+7:])
+ }
+ imageLayers = append(imageLayers, cmd)
+ }
+
+ metadata := &Metadata{
+ Type: TypeOCI,
+ Platform: platform,
+ Licenses: image.Config.Labels[labelLicenses],
+ ProjectURL: image.Config.Labels[labelURL],
+ RepositoryURL: image.Config.Labels[labelSource],
+ DocumentationURL: image.Config.Labels[labelDocumentation],
+ Description: image.Config.Labels[labelDescription],
+ Labels: image.Config.Labels,
+ ImageLayers: imageLayers,
+ }
+
+ if authors, ok := image.Config.Labels[labelAuthors]; ok {
+ metadata.Authors = []string{authors}
+ }
+
+ if !validation.IsValidURL(metadata.ProjectURL) {
+ metadata.ProjectURL = ""
+ }
+ if !validation.IsValidURL(metadata.RepositoryURL) {
+ metadata.RepositoryURL = ""
+ }
+ if !validation.IsValidURL(metadata.DocumentationURL) {
+ metadata.DocumentationURL = ""
+ }
+
+ return metadata, nil
+}
+
+func parseHelmConfig(r io.Reader) (*Metadata, error) {
+ var config helm.Metadata
+ if err := json.NewDecoder(r).Decode(&config); err != nil {
+ return nil, err
+ }
+
+ metadata := &Metadata{
+ Type: TypeHelm,
+ Description: config.Description,
+ ProjectURL: config.Home,
+ }
+
+ if len(config.Maintainers) > 0 {
+ authors := make([]string, 0, len(config.Maintainers))
+ for _, maintainer := range config.Maintainers {
+ authors = append(authors, maintainer.Name)
+ }
+ metadata.Authors = authors
+ }
+
+ if len(config.Sources) > 0 && validation.IsValidURL(config.Sources[0]) {
+ metadata.RepositoryURL = config.Sources[0]
+ }
+ if !validation.IsValidURL(metadata.ProjectURL) {
+ metadata.ProjectURL = ""
+ }
+
+ return metadata, nil
+}
diff --git a/modules/packages/container/metadata_test.go b/modules/packages/container/metadata_test.go
new file mode 100644
index 0000000000..9400cf6954
--- /dev/null
+++ b/modules/packages/container/metadata_test.go
@@ -0,0 +1,62 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package container
+
+import (
+ "strings"
+ "testing"
+
+ "code.gitea.io/gitea/modules/packages/container/helm"
+ "code.gitea.io/gitea/modules/packages/container/oci"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestParseImageConfig(t *testing.T) {
+ description := "Image Description"
+ author := "Gitea"
+ license := "MIT"
+ projectURL := "https://gitea.io"
+ repositoryURL := "https://gitea.com/gitea"
+ documentationURL := "https://docs.gitea.io"
+
+ configOCI := `{"config": {"labels": {"` + labelAuthors + `": "` + author + `", "` + labelLicenses + `": "` + license + `", "` + labelURL + `": "` + projectURL + `", "` + labelSource + `": "` + repositoryURL + `", "` + labelDocumentation + `": "` + documentationURL + `", "` + labelDescription + `": "` + description + `"}}, "history": [{"created_by": "do it 1"}, {"created_by": "dummy #(nop) do it 2"}]}`
+
+ metadata, err := ParseImageConfig(oci.MediaType(oci.MediaTypeImageManifest), strings.NewReader(configOCI))
+ assert.NoError(t, err)
+
+ assert.Equal(t, TypeOCI, metadata.Type)
+ assert.Equal(t, description, metadata.Description)
+ assert.ElementsMatch(t, []string{author}, metadata.Authors)
+ assert.Equal(t, license, metadata.Licenses)
+ assert.Equal(t, projectURL, metadata.ProjectURL)
+ assert.Equal(t, repositoryURL, metadata.RepositoryURL)
+ assert.Equal(t, documentationURL, metadata.DocumentationURL)
+ assert.Equal(t, []string{"do it 1", "do it 2"}, metadata.ImageLayers)
+ assert.Equal(
+ t,
+ map[string]string{
+ labelAuthors: author,
+ labelLicenses: license,
+ labelURL: projectURL,
+ labelSource: repositoryURL,
+ labelDocumentation: documentationURL,
+ labelDescription: description,
+ },
+ metadata.Labels,
+ )
+ assert.Empty(t, metadata.MultiArch)
+
+ configHelm := `{"description":"` + description + `", "home": "` + projectURL + `", "sources": ["` + repositoryURL + `"], "maintainers":[{"name":"` + author + `"}]}`
+
+ metadata, err = ParseImageConfig(oci.MediaType(helm.ConfigMediaType), strings.NewReader(configHelm))
+ assert.NoError(t, err)
+
+ assert.Equal(t, TypeHelm, metadata.Type)
+ assert.Equal(t, description, metadata.Description)
+ assert.ElementsMatch(t, []string{author}, metadata.Authors)
+ assert.Equal(t, projectURL, metadata.ProjectURL)
+ assert.Equal(t, repositoryURL, metadata.RepositoryURL)
+}
diff --git a/modules/packages/container/oci/digest.go b/modules/packages/container/oci/digest.go
new file mode 100644
index 0000000000..5234814cfe
--- /dev/null
+++ b/modules/packages/container/oci/digest.go
@@ -0,0 +1,27 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package oci
+
+import (
+ "regexp"
+ "strings"
+)
+
+var digestPattern = regexp.MustCompile(`\Asha256:[a-f0-9]{64}\z`)
+
+type Digest string
+
+// Validate checks if the digest has a valid SHA256 signature
+func (d Digest) Validate() bool {
+ return digestPattern.MatchString(string(d))
+}
+
+func (d Digest) Hash() string {
+ p := strings.SplitN(string(d), ":", 2)
+ if len(p) != 2 {
+ return ""
+ }
+ return p[1]
+}
diff --git a/modules/packages/container/oci/mediatype.go b/modules/packages/container/oci/mediatype.go
new file mode 100644
index 0000000000..2636fbe288
--- /dev/null
+++ b/modules/packages/container/oci/mediatype.go
@@ -0,0 +1,36 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package oci
+
+import (
+ "strings"
+)
+
+const (
+ MediaTypeImageManifest = "application/vnd.oci.image.manifest.v1+json"
+ MediaTypeImageIndex = "application/vnd.oci.image.index.v1+json"
+ MediaTypeDockerManifest = "application/vnd.docker.distribution.manifest.v2+json"
+ MediaTypeDockerManifestList = "application/vnd.docker.distribution.manifest.list.v2+json"
+)
+
+type MediaType string
+
+// IsValid tests if the media type is in the OCI or Docker namespace
+func (m MediaType) IsValid() bool {
+ s := string(m)
+ return strings.HasPrefix(s, "application/vnd.docker.") || strings.HasPrefix(s, "application/vnd.oci.")
+}
+
+// IsImageManifest tests if the media type is an image manifest
+func (m MediaType) IsImageManifest() bool {
+ s := string(m)
+ return strings.EqualFold(s, MediaTypeDockerManifest) || strings.EqualFold(s, MediaTypeImageManifest)
+}
+
+// IsImageIndex tests if the media type is an image index
+func (m MediaType) IsImageIndex() bool {
+ s := string(m)
+ return strings.EqualFold(s, MediaTypeDockerManifestList) || strings.EqualFold(s, MediaTypeImageIndex)
+}
diff --git a/modules/packages/container/oci/oci.go b/modules/packages/container/oci/oci.go
new file mode 100644
index 0000000000..01cca8fe69
--- /dev/null
+++ b/modules/packages/container/oci/oci.go
@@ -0,0 +1,191 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package oci
+
+import (
+ "time"
+)
+
+// https://github.com/opencontainers/image-spec/tree/main/specs-go/v1
+
+// ImageConfig defines the execution parameters which should be used as a base when running a container using an image.
+type ImageConfig struct {
+ // User defines the username or UID which the process in the container should run as.
+ User string `json:"User,omitempty"`
+
+ // ExposedPorts a set of ports to expose from a container running this image.
+ ExposedPorts map[string]struct{} `json:"ExposedPorts,omitempty"`
+
+ // Env is a list of environment variables to be used in a container.
+ Env []string `json:"Env,omitempty"`
+
+ // Entrypoint defines a list of arguments to use as the command to execute when the container starts.
+ Entrypoint []string `json:"Entrypoint,omitempty"`
+
+ // Cmd defines the default arguments to the entrypoint of the container.
+ Cmd []string `json:"Cmd,omitempty"`
+
+ // Volumes is a set of directories describing where the process is likely write data specific to a container instance.
+ Volumes map[string]struct{} `json:"Volumes,omitempty"`
+
+ // WorkingDir sets the current working directory of the entrypoint process in the container.
+ WorkingDir string `json:"WorkingDir,omitempty"`
+
+ // Labels contains arbitrary metadata for the container.
+ Labels map[string]string `json:"Labels,omitempty"`
+
+ // StopSignal contains the system call signal that will be sent to the container to exit.
+ StopSignal string `json:"StopSignal,omitempty"`
+}
+
+// RootFS describes a layer content addresses
+type RootFS struct {
+ // Type is the type of the rootfs.
+ Type string `json:"type"`
+
+ // DiffIDs is an array of layer content hashes, in order from bottom-most to top-most.
+ DiffIDs []string `json:"diff_ids"`
+}
+
+// History describes the history of a layer.
+type History struct {
+ // Created is the combined date and time at which the layer was created, formatted as defined by RFC 3339, section 5.6.
+ Created *time.Time `json:"created,omitempty"`
+
+ // CreatedBy is the command which created the layer.
+ CreatedBy string `json:"created_by,omitempty"`
+
+ // Author is the author of the build point.
+ Author string `json:"author,omitempty"`
+
+ // Comment is a custom message set when creating the layer.
+ Comment string `json:"comment,omitempty"`
+
+ // EmptyLayer is used to mark if the history item created a filesystem diff.
+ EmptyLayer bool `json:"empty_layer,omitempty"`
+}
+
+// Image is the JSON structure which describes some basic information about the image.
+// This provides the `application/vnd.oci.image.config.v1+json` mediatype when marshalled to JSON.
+type Image struct {
+ // Created is the combined date and time at which the image was created, formatted as defined by RFC 3339, section 5.6.
+ Created *time.Time `json:"created,omitempty"`
+
+ // Author defines the name and/or email address of the person or entity which created and is responsible for maintaining the image.
+ Author string `json:"author,omitempty"`
+
+ // Architecture is the CPU architecture which the binaries in this image are built to run on.
+ Architecture string `json:"architecture"`
+
+ // Variant is the variant of the specified CPU architecture which image binaries are intended to run on.
+ Variant string `json:"variant,omitempty"`
+
+ // OS is the name of the operating system which the image is built to run on.
+ OS string `json:"os"`
+
+ // OSVersion is an optional field specifying the operating system
+ // version, for example on Windows `10.0.14393.1066`.
+ OSVersion string `json:"os.version,omitempty"`
+
+ // OSFeatures is an optional field specifying an array of strings,
+ // each listing a required OS feature (for example on Windows `win32k`).
+ OSFeatures []string `json:"os.features,omitempty"`
+
+ // Config defines the execution parameters which should be used as a base when running a container using the image.
+ Config ImageConfig `json:"config,omitempty"`
+
+ // RootFS references the layer content addresses used by the image.
+ RootFS RootFS `json:"rootfs"`
+
+ // History describes the history of each layer.
+ History []History `json:"history,omitempty"`
+}
+
+// Descriptor describes the disposition of targeted content.
+// This structure provides `application/vnd.oci.descriptor.v1+json` mediatype
+// when marshalled to JSON.
+type Descriptor struct {
+ // MediaType is the media type of the object this schema refers to.
+ MediaType MediaType `json:"mediaType,omitempty"`
+
+ // Digest is the digest of the targeted content.
+ Digest Digest `json:"digest"`
+
+ // Size specifies the size in bytes of the blob.
+ Size int64 `json:"size"`
+
+ // URLs specifies a list of URLs from which this object MAY be downloaded
+ URLs []string `json:"urls,omitempty"`
+
+ // Annotations contains arbitrary metadata relating to the targeted content.
+ Annotations map[string]string `json:"annotations,omitempty"`
+
+ // Data is an embedding of the targeted content. This is encoded as a base64
+ // string when marshalled to JSON (automatically, by encoding/json). If
+ // present, Data can be used directly to avoid fetching the targeted content.
+ Data []byte `json:"data,omitempty"`
+
+ // Platform describes the platform which the image in the manifest runs on.
+ //
+ // This should only be used when referring to a manifest.
+ Platform *Platform `json:"platform,omitempty"`
+}
+
+// Platform describes the platform which the image in the manifest runs on.
+type Platform struct {
+ // Architecture field specifies the CPU architecture, for example
+ // `amd64` or `ppc64`.
+ Architecture string `json:"architecture"`
+
+ // OS specifies the operating system, for example `linux` or `windows`.
+ OS string `json:"os"`
+
+ // OSVersion is an optional field specifying the operating system
+ // version, for example on Windows `10.0.14393.1066`.
+ OSVersion string `json:"os.version,omitempty"`
+
+ // OSFeatures is an optional field specifying an array of strings,
+ // each listing a required OS feature (for example on Windows `win32k`).
+ OSFeatures []string `json:"os.features,omitempty"`
+
+ // Variant is an optional field specifying a variant of the CPU, for
+ // example `v7` to specify ARMv7 when architecture is `arm`.
+ Variant string `json:"variant,omitempty"`
+}
+
+type SchemaMediaBase struct {
+ // SchemaVersion is the image manifest schema that this image follows
+ SchemaVersion int `json:"schemaVersion"`
+
+ // MediaType specifies the type of this document data structure e.g. `application/vnd.oci.image.manifest.v1+json`
+ MediaType MediaType `json:"mediaType,omitempty"`
+}
+
+// Manifest provides `application/vnd.oci.image.manifest.v1+json` mediatype structure when marshalled to JSON.
+type Manifest struct {
+ SchemaMediaBase
+
+ // Config references a configuration object for a container, by digest.
+ // The referenced configuration object is a JSON blob that the runtime uses to set up the container.
+ Config Descriptor `json:"config"`
+
+ // Layers is an indexed list of layers referenced by the manifest.
+ Layers []Descriptor `json:"layers"`
+
+ // Annotations contains arbitrary metadata for the image manifest.
+ Annotations map[string]string `json:"annotations,omitempty"`
+}
+
+// Index references manifests for various platforms.
+// This structure provides `application/vnd.oci.image.index.v1+json` mediatype when marshalled to JSON.
+type Index struct {
+ SchemaMediaBase
+
+ // Manifests references platform specific manifests.
+ Manifests []Descriptor `json:"manifests"`
+
+ // Annotations contains arbitrary metadata for the image index.
+ Annotations map[string]string `json:"annotations,omitempty"`
+}
diff --git a/modules/packages/container/oci/reference.go b/modules/packages/container/oci/reference.go
new file mode 100644
index 0000000000..120ff122d4
--- /dev/null
+++ b/modules/packages/container/oci/reference.go
@@ -0,0 +1,17 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package oci
+
+import (
+ "regexp"
+)
+
+var referencePattern = regexp.MustCompile(`\A[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}\z`)
+
+type Reference string
+
+func (r Reference) Validate() bool {
+ return referencePattern.MatchString(string(r))
+}
diff --git a/modules/packages/content_store.go b/modules/packages/content_store.go
new file mode 100644
index 0000000000..64c3eedc23
--- /dev/null
+++ b/modules/packages/content_store.go
@@ -0,0 +1,47 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package packages
+
+import (
+ "io"
+ "path"
+
+ "code.gitea.io/gitea/modules/storage"
+)
+
+// BlobHash256Key is the key to address a blob content
+type BlobHash256Key string
+
+// ContentStore is a wrapper around ObjectStorage
+type ContentStore struct {
+ store storage.ObjectStorage
+}
+
+// NewContentStore creates the default package store
+func NewContentStore() *ContentStore {
+ contentStore := &ContentStore{storage.Packages}
+ return contentStore
+}
+
+// Get gets a package blob
+func (s *ContentStore) Get(key BlobHash256Key) (storage.Object, error) {
+ return s.store.Open(keyToRelativePath(key))
+}
+
+// Save stores a package blob
+func (s *ContentStore) Save(key BlobHash256Key, r io.Reader, size int64) error {
+ _, err := s.store.Save(keyToRelativePath(key), r, size)
+ return err
+}
+
+// Delete deletes a package blob
+func (s *ContentStore) Delete(key BlobHash256Key) error {
+ return s.store.Delete(keyToRelativePath(key))
+}
+
+// keyToRelativePath converts the sha256 key aabb000000... to aa/bb/aabb000000...
+func keyToRelativePath(key BlobHash256Key) string {
+ return path.Join(string(key)[0:2], string(key)[2:4], string(key))
+}
diff --git a/modules/packages/hashed_buffer.go b/modules/packages/hashed_buffer.go
new file mode 100644
index 0000000000..3f8cafcfb5
--- /dev/null
+++ b/modules/packages/hashed_buffer.go
@@ -0,0 +1,70 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package packages
+
+import (
+ "io"
+
+ "code.gitea.io/gitea/modules/util/filebuffer"
+)
+
+// HashedSizeReader provide methods to read, sum hashes and a Size method
+type HashedSizeReader interface {
+ io.Reader
+ HashSummer
+ Size() int64
+}
+
+// HashedBuffer is buffer which calculates multiple checksums
+type HashedBuffer struct {
+ *filebuffer.FileBackedBuffer
+
+ hash *MultiHasher
+
+ combinedWriter io.Writer
+}
+
+// NewHashedBuffer creates a hashed buffer with a specific maximum memory size
+func NewHashedBuffer(maxMemorySize int) (*HashedBuffer, error) {
+ b, err := filebuffer.New(maxMemorySize)
+ if err != nil {
+ return nil, err
+ }
+
+ hash := NewMultiHasher()
+
+ combinedWriter := io.MultiWriter(b, hash)
+
+ return &HashedBuffer{
+ b,
+ hash,
+ combinedWriter,
+ }, nil
+}
+
+// CreateHashedBufferFromReader creates a hashed buffer and copies the provided reader data into it.
+func CreateHashedBufferFromReader(r io.Reader, maxMemorySize int) (*HashedBuffer, error) {
+ b, err := NewHashedBuffer(maxMemorySize)
+ if err != nil {
+ return nil, err
+ }
+
+ _, err = io.Copy(b, r)
+ if err != nil {
+ return nil, err
+ }
+
+ return b, nil
+}
+
+// Write implements io.Writer
+func (b *HashedBuffer) Write(p []byte) (int, error) {
+ return b.combinedWriter.Write(p)
+}
+
+// Sums gets the MD5, SHA1, SHA256 and SHA512 checksums of the data
+func (b *HashedBuffer) Sums() (hashMD5, hashSHA1, hashSHA256, hashSHA512 []byte) {
+ return b.hash.Sums()
+}
diff --git a/modules/packages/maven/metadata.go b/modules/packages/maven/metadata.go
new file mode 100644
index 0000000000..6ee9d69687
--- /dev/null
+++ b/modules/packages/maven/metadata.go
@@ -0,0 +1,89 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package maven
+
+import (
+ "encoding/xml"
+ "io"
+
+ "code.gitea.io/gitea/modules/validation"
+)
+
+// Metadata represents the metadata of a Maven package
+type Metadata struct {
+ GroupID string `json:"group_id,omitempty"`
+ ArtifactID string `json:"artifact_id,omitempty"`
+ Name string `json:"name,omitempty"`
+ Description string `json:"description,omitempty"`
+ ProjectURL string `json:"project_url,omitempty"`
+ Licenses []string `json:"licenses,omitempty"`
+ Dependencies []*Dependency `json:"dependencies,omitempty"`
+}
+
+// Dependency represents a dependency of a Maven package
+type Dependency struct {
+ GroupID string `json:"group_id,omitempty"`
+ ArtifactID string `json:"artifact_id,omitempty"`
+ Version string `json:"version,omitempty"`
+}
+
+type pomStruct struct {
+ XMLName xml.Name `xml:"project"`
+ GroupID string `xml:"groupId"`
+ ArtifactID string `xml:"artifactId"`
+ Version string `xml:"version"`
+ Name string `xml:"name"`
+ Description string `xml:"description"`
+ URL string `xml:"url"`
+ Licenses []struct {
+ Name string `xml:"name"`
+ URL string `xml:"url"`
+ Distribution string `xml:"distribution"`
+ } `xml:"licenses>license"`
+ Dependencies []struct {
+ GroupID string `xml:"groupId"`
+ ArtifactID string `xml:"artifactId"`
+ Version string `xml:"version"`
+ Scope string `xml:"scope"`
+ } `xml:"dependencies>dependency"`
+}
+
+// ParsePackageMetaData parses the metadata of a pom file
+func ParsePackageMetaData(r io.Reader) (*Metadata, error) {
+ var pom pomStruct
+ if err := xml.NewDecoder(r).Decode(&pom); err != nil {
+ return nil, err
+ }
+
+ if !validation.IsValidURL(pom.URL) {
+ pom.URL = ""
+ }
+
+ licenses := make([]string, 0, len(pom.Licenses))
+ for _, l := range pom.Licenses {
+ if l.Name != "" {
+ licenses = append(licenses, l.Name)
+ }
+ }
+
+ dependencies := make([]*Dependency, 0, len(pom.Dependencies))
+ for _, d := range pom.Dependencies {
+ dependencies = append(dependencies, &Dependency{
+ GroupID: d.GroupID,
+ ArtifactID: d.ArtifactID,
+ Version: d.Version,
+ })
+ }
+
+ return &Metadata{
+ GroupID: pom.GroupID,
+ ArtifactID: pom.ArtifactID,
+ Name: pom.Name,
+ Description: pom.Description,
+ ProjectURL: pom.URL,
+ Licenses: licenses,
+ Dependencies: dependencies,
+ }, nil
+}
diff --git a/modules/packages/maven/metadata_test.go b/modules/packages/maven/metadata_test.go
new file mode 100644
index 0000000000..a17d456560
--- /dev/null
+++ b/modules/packages/maven/metadata_test.go
@@ -0,0 +1,73 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package maven
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+const (
+ groupID = "org.gitea"
+ artifactID = "my-project"
+ version = "1.0.1"
+ name = "My Gitea Project"
+ description = "Package Description"
+ projectURL = "https://gitea.io"
+ license = "MIT"
+ dependencyGroupID = "org.gitea.core"
+ dependencyArtifactID = "git"
+ dependencyVersion = "5.0.0"
+)
+
+const pomContent = `<?xml version="1.0"?>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <groupId>` + groupID + `</groupId>
+ <artifactId>` + artifactID + `</artifactId>
+ <version>` + version + `</version>
+ <name>` + name + `</name>
+ <description>` + description + `</description>
+ <url>` + projectURL + `</url>
+ <licenses>
+ <license>
+ <name>` + license + `</name>
+ </license>
+ </licenses>
+ <dependencies>
+ <dependency>
+ <groupId>` + dependencyGroupID + `</groupId>
+ <artifactId>` + dependencyArtifactID + `</artifactId>
+ <version>` + dependencyVersion + `</version>
+ </dependency>
+ </dependencies>
+</project>`
+
+func TestParsePackageMetaData(t *testing.T) {
+ t.Run("InvalidFile", func(t *testing.T) {
+ m, err := ParsePackageMetaData(strings.NewReader(""))
+ assert.Nil(t, m)
+ assert.Error(t, err)
+ })
+
+ t.Run("Valid", func(t *testing.T) {
+ m, err := ParsePackageMetaData(strings.NewReader(pomContent))
+ assert.NoError(t, err)
+ assert.NotNil(t, m)
+
+ assert.Equal(t, groupID, m.GroupID)
+ assert.Equal(t, artifactID, m.ArtifactID)
+ assert.Equal(t, name, m.Name)
+ assert.Equal(t, description, m.Description)
+ assert.Equal(t, projectURL, m.ProjectURL)
+ assert.Len(t, m.Licenses, 1)
+ assert.Equal(t, license, m.Licenses[0])
+ assert.Len(t, m.Dependencies, 1)
+ assert.Equal(t, dependencyGroupID, m.Dependencies[0].GroupID)
+ assert.Equal(t, dependencyArtifactID, m.Dependencies[0].ArtifactID)
+ assert.Equal(t, dependencyVersion, m.Dependencies[0].Version)
+ })
+}
diff --git a/modules/packages/multi_hasher.go b/modules/packages/multi_hasher.go
new file mode 100644
index 0000000000..0659a18d2a
--- /dev/null
+++ b/modules/packages/multi_hasher.go
@@ -0,0 +1,123 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package packages
+
+import (
+ "crypto/md5"
+ "crypto/sha1"
+ "crypto/sha256"
+ "crypto/sha512"
+ "encoding"
+ "errors"
+ "hash"
+ "io"
+)
+
+const (
+ marshaledSizeMD5 = 92
+ marshaledSizeSHA1 = 96
+ marshaledSizeSHA256 = 108
+ marshaledSizeSHA512 = 204
+
+ marshaledSize = marshaledSizeMD5 + marshaledSizeSHA1 + marshaledSizeSHA256 + marshaledSizeSHA512
+)
+
+// HashSummer provide a Sums method
+type HashSummer interface {
+ Sums() (hashMD5, hashSHA1, hashSHA256, hashSHA512 []byte)
+}
+
+// MultiHasher calculates multiple checksums
+type MultiHasher struct {
+ md5 hash.Hash
+ sha1 hash.Hash
+ sha256 hash.Hash
+ sha512 hash.Hash
+
+ combinedWriter io.Writer
+}
+
+// NewMultiHasher creates a multi hasher
+func NewMultiHasher() *MultiHasher {
+ md5 := md5.New()
+ sha1 := sha1.New()
+ sha256 := sha256.New()
+ sha512 := sha512.New()
+
+ combinedWriter := io.MultiWriter(md5, sha1, sha256, sha512)
+
+ return &MultiHasher{
+ md5,
+ sha1,
+ sha256,
+ sha512,
+ combinedWriter,
+ }
+}
+
+// MarshalBinary implements encoding.BinaryMarshaler
+func (h *MultiHasher) MarshalBinary() ([]byte, error) {
+ md5Bytes, err := h.md5.(encoding.BinaryMarshaler).MarshalBinary()
+ if err != nil {
+ return nil, err
+ }
+ sha1Bytes, err := h.sha1.(encoding.BinaryMarshaler).MarshalBinary()
+ if err != nil {
+ return nil, err
+ }
+ sha256Bytes, err := h.sha256.(encoding.BinaryMarshaler).MarshalBinary()
+ if err != nil {
+ return nil, err
+ }
+ sha512Bytes, err := h.sha512.(encoding.BinaryMarshaler).MarshalBinary()
+ if err != nil {
+ return nil, err
+ }
+
+ b := make([]byte, 0, marshaledSize)
+ b = append(b, md5Bytes...)
+ b = append(b, sha1Bytes...)
+ b = append(b, sha256Bytes...)
+ b = append(b, sha512Bytes...)
+ return b, nil
+}
+
+// UnmarshalBinary implements encoding.BinaryUnmarshaler
+func (h *MultiHasher) UnmarshalBinary(b []byte) error {
+ if len(b) != marshaledSize {
+ return errors.New("invalid hash state size")
+ }
+
+ if err := h.md5.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[:marshaledSizeMD5]); err != nil {
+ return err
+ }
+
+ b = b[marshaledSizeMD5:]
+ if err := h.sha1.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[:marshaledSizeSHA1]); err != nil {
+ return err
+ }
+
+ b = b[marshaledSizeSHA1:]
+ if err := h.sha256.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[:marshaledSizeSHA256]); err != nil {
+ return err
+ }
+
+ b = b[marshaledSizeSHA256:]
+ return h.sha512.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[:marshaledSizeSHA512])
+}
+
+// Write implements io.Writer
+func (h *MultiHasher) Write(p []byte) (int, error) {
+ return h.combinedWriter.Write(p)
+}
+
+// Sums gets the MD5, SHA1, SHA256 and SHA512 checksums of the data
+func (h *MultiHasher) Sums() (hashMD5, hashSHA1, hashSHA256, hashSHA512 []byte) {
+ hashMD5 = h.md5.Sum(nil)
+ hashSHA1 = h.sha1.Sum(nil)
+ hashSHA256 = h.sha256.Sum(nil)
+ hashSHA512 = h.sha512.Sum(nil)
+ return
+}
diff --git a/modules/packages/multi_hasher_test.go b/modules/packages/multi_hasher_test.go
new file mode 100644
index 0000000000..6c895ce120
--- /dev/null
+++ b/modules/packages/multi_hasher_test.go
@@ -0,0 +1,54 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package packages
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+const (
+ expectedMD5 = "e3bef03c5f3b7f6b3ab3e3053ed71e9c"
+ expectedSHA1 = "060b3b99f88e96085b4a68e095bc9e3d1d91e1bc"
+ expectedSHA256 = "6ccce4863b70f258d691f59609d31b4502e1ba5199942d3bc5d35d17a4ce771d"
+ expectedSHA512 = "7f70e439ba8c52025c1f06cdf6ae443c4b8ed2e90059cdb9bbbf8adf80846f185a24acca9245b128b226d61753b0d7ed46580a69c8999eeff3bc13a4d0bd816c"
+)
+
+func TestMultiHasherSums(t *testing.T) {
+ t.Run("Sums", func(t *testing.T) {
+ h := NewMultiHasher()
+ h.Write([]byte("gitea"))
+
+ hashMD5, hashSHA1, hashSHA256, hashSHA512 := h.Sums()
+
+ assert.Equal(t, expectedMD5, fmt.Sprintf("%x", hashMD5))
+ assert.Equal(t, expectedSHA1, fmt.Sprintf("%x", hashSHA1))
+ assert.Equal(t, expectedSHA256, fmt.Sprintf("%x", hashSHA256))
+ assert.Equal(t, expectedSHA512, fmt.Sprintf("%x", hashSHA512))
+ })
+
+ t.Run("State", func(t *testing.T) {
+ h := NewMultiHasher()
+ h.Write([]byte("git"))
+
+ state, err := h.MarshalBinary()
+ assert.NoError(t, err)
+
+ h2 := NewMultiHasher()
+ err = h2.UnmarshalBinary(state)
+ assert.NoError(t, err)
+
+ h2.Write([]byte("ea"))
+
+ hashMD5, hashSHA1, hashSHA256, hashSHA512 := h2.Sums()
+
+ assert.Equal(t, expectedMD5, fmt.Sprintf("%x", hashMD5))
+ assert.Equal(t, expectedSHA1, fmt.Sprintf("%x", hashSHA1))
+ assert.Equal(t, expectedSHA256, fmt.Sprintf("%x", hashSHA256))
+ assert.Equal(t, expectedSHA512, fmt.Sprintf("%x", hashSHA512))
+ })
+}
diff --git a/modules/packages/npm/creator.go b/modules/packages/npm/creator.go
new file mode 100644
index 0000000000..88ce55ecdb
--- /dev/null
+++ b/modules/packages/npm/creator.go
@@ -0,0 +1,256 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package npm
+
+import (
+ "bytes"
+ "crypto/sha1"
+ "crypto/sha512"
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "io"
+ "regexp"
+ "strings"
+ "time"
+
+ "code.gitea.io/gitea/modules/json"
+ "code.gitea.io/gitea/modules/validation"
+
+ "github.com/hashicorp/go-version"
+)
+
+var (
+ // ErrInvalidPackage indicates an invalid package
+ ErrInvalidPackage = errors.New("The package is invalid")
+ // ErrInvalidPackageName indicates an invalid name
+ ErrInvalidPackageName = errors.New("The package name is invalid")
+ // ErrInvalidPackageVersion indicates an invalid version
+ ErrInvalidPackageVersion = errors.New("The package version is invalid")
+ // ErrInvalidAttachment indicates a invalid attachment
+ ErrInvalidAttachment = errors.New("The package attachment is invalid")
+ // ErrInvalidIntegrity indicates an integrity validation error
+ ErrInvalidIntegrity = errors.New("Failed to validate integrity")
+)
+
+var nameMatch = regexp.MustCompile(`\A((@[^\s\/~'!\(\)\*]+?)[\/])?([^_.][^\s\/~'!\(\)\*]+)\z`)
+
+// Package represents a npm package
+type Package struct {
+ Name string
+ Version string
+ DistTags []string
+ Metadata Metadata
+ Filename string
+ Data []byte
+}
+
+// PackageMetadata https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#package
+type PackageMetadata struct {
+ ID string `json:"_id"`
+ Name string `json:"name"`
+ Description string `json:"description"`
+ DistTags map[string]string `json:"dist-tags,omitempty"`
+ Versions map[string]*PackageMetadataVersion `json:"versions"`
+ Readme string `json:"readme,omitempty"`
+ Maintainers []User `json:"maintainers,omitempty"`
+ Time map[string]time.Time `json:"time,omitempty"`
+ Homepage string `json:"homepage,omitempty"`
+ Keywords []string `json:"keywords,omitempty"`
+ Repository Repository `json:"repository,omitempty"`
+ Author User `json:"author"`
+ ReadmeFilename string `json:"readmeFilename,omitempty"`
+ Users map[string]bool `json:"users,omitempty"`
+ License string `json:"license,omitempty"`
+}
+
+// PackageMetadataVersion https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version
+type PackageMetadataVersion struct {
+ ID string `json:"_id"`
+ Name string `json:"name"`
+ Version string `json:"version"`
+ Description string `json:"description"`
+ Author User `json:"author"`
+ Homepage string `json:"homepage,omitempty"`
+ License string `json:"license,omitempty"`
+ Repository Repository `json:"repository,omitempty"`
+ Keywords []string `json:"keywords,omitempty"`
+ Dependencies map[string]string `json:"dependencies,omitempty"`
+ DevDependencies map[string]string `json:"devDependencies,omitempty"`
+ PeerDependencies map[string]string `json:"peerDependencies,omitempty"`
+ OptionalDependencies map[string]string `json:"optionalDependencies,omitempty"`
+ Readme string `json:"readme,omitempty"`
+ Dist PackageDistribution `json:"dist"`
+ Maintainers []User `json:"maintainers,omitempty"`
+}
+
+// PackageDistribution https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version
+type PackageDistribution struct {
+ Integrity string `json:"integrity"`
+ Shasum string `json:"shasum"`
+ Tarball string `json:"tarball"`
+ FileCount int `json:"fileCount,omitempty"`
+ UnpackedSize int `json:"unpackedSize,omitempty"`
+ NpmSignature string `json:"npm-signature,omitempty"`
+}
+
+// User https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#package
+type User struct {
+ Username string `json:"username,omitempty"`
+ Name string `json:"name"`
+ Email string `json:"email,omitempty"`
+ URL string `json:"url,omitempty"`
+}
+
+// UnmarshalJSON is needed because User objects can be strings or objects
+func (u *User) UnmarshalJSON(data []byte) error {
+ switch data[0] {
+ case '"':
+ if err := json.Unmarshal(data, &u.Name); err != nil {
+ return err
+ }
+ case '{':
+ var tmp struct {
+ Username string `json:"username"`
+ Name string `json:"name"`
+ Email string `json:"email"`
+ URL string `json:"url"`
+ }
+ if err := json.Unmarshal(data, &tmp); err != nil {
+ return err
+ }
+ u.Username = tmp.Username
+ u.Name = tmp.Name
+ u.Email = tmp.Email
+ u.URL = tmp.URL
+ }
+ return nil
+}
+
+// Repository https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version
+type Repository struct {
+ Type string `json:"type"`
+ URL string `json:"url"`
+}
+
+// PackageAttachment https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#package
+type PackageAttachment struct {
+ ContentType string `json:"content_type"`
+ Data string `json:"data"`
+ Length int `json:"length"`
+}
+
+type packageUpload struct {
+ PackageMetadata
+ Attachments map[string]*PackageAttachment `json:"_attachments"`
+}
+
+// ParsePackage parses the content into a npm package
+func ParsePackage(r io.Reader) (*Package, error) {
+ var upload packageUpload
+ if err := json.NewDecoder(r).Decode(&upload); err != nil {
+ return nil, err
+ }
+
+ for _, meta := range upload.Versions {
+ if !validateName(meta.Name) {
+ return nil, ErrInvalidPackageName
+ }
+
+ v, err := version.NewSemver(meta.Version)
+ if err != nil {
+ return nil, ErrInvalidPackageVersion
+ }
+
+ scope := ""
+ name := meta.Name
+ nameParts := strings.SplitN(meta.Name, "/", 2)
+ if len(nameParts) == 2 {
+ scope = nameParts[0]
+ name = nameParts[1]
+ }
+
+ if !validation.IsValidURL(meta.Homepage) {
+ meta.Homepage = ""
+ }
+
+ p := &Package{
+ Name: meta.Name,
+ Version: v.String(),
+ DistTags: make([]string, 0, 1),
+ Metadata: Metadata{
+ Scope: scope,
+ Name: name,
+ Description: meta.Description,
+ Author: meta.Author.Name,
+ License: meta.License,
+ ProjectURL: meta.Homepage,
+ Keywords: meta.Keywords,
+ Dependencies: meta.Dependencies,
+ DevelopmentDependencies: meta.DevDependencies,
+ PeerDependencies: meta.PeerDependencies,
+ OptionalDependencies: meta.OptionalDependencies,
+ Readme: meta.Readme,
+ },
+ }
+
+ for tag := range upload.DistTags {
+ p.DistTags = append(p.DistTags, tag)
+ }
+
+ p.Filename = strings.ToLower(fmt.Sprintf("%s-%s.tgz", name, p.Version))
+
+ attachment := func() *PackageAttachment {
+ for _, a := range upload.Attachments {
+ return a
+ }
+ return nil
+ }()
+ if attachment == nil || len(attachment.Data) == 0 {
+ return nil, ErrInvalidAttachment
+ }
+
+ data, err := base64.StdEncoding.DecodeString(attachment.Data)
+ if err != nil {
+ return nil, ErrInvalidAttachment
+ }
+ p.Data = data
+
+ integrity := strings.SplitN(meta.Dist.Integrity, "-", 2)
+ if len(integrity) != 2 {
+ return nil, ErrInvalidIntegrity
+ }
+ integrityHash, err := base64.StdEncoding.DecodeString(integrity[1])
+ if err != nil {
+ return nil, ErrInvalidIntegrity
+ }
+ var hash []byte
+ switch integrity[0] {
+ case "sha1":
+ tmp := sha1.Sum(data)
+ hash = tmp[:]
+ case "sha512":
+ tmp := sha512.Sum512(data)
+ hash = tmp[:]
+ }
+ if !bytes.Equal(integrityHash, hash) {
+ return nil, ErrInvalidIntegrity
+ }
+
+ return p, nil
+ }
+
+ return nil, ErrInvalidPackage
+}
+
+func validateName(name string) bool {
+ if strings.TrimSpace(name) != name {
+ return false
+ }
+ if len(name) == 0 || len(name) > 214 {
+ return false
+ }
+ return nameMatch.MatchString(name)
+}
diff --git a/modules/packages/npm/creator_test.go b/modules/packages/npm/creator_test.go
new file mode 100644
index 0000000000..64ae6238f3
--- /dev/null
+++ b/modules/packages/npm/creator_test.go
@@ -0,0 +1,272 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package npm
+
+import (
+ "bytes"
+ "encoding/base64"
+ "fmt"
+ "strings"
+ "testing"
+
+ "code.gitea.io/gitea/modules/json"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestParsePackage(t *testing.T) {
+ packageScope := "@scope"
+ packageName := "test-package"
+ packageFullName := packageScope + "/" + packageName
+ packageVersion := "1.0.1-pre"
+ packageTag := "latest"
+ packageAuthor := "KN4CK3R"
+ packageDescription := "Test Description"
+ data := "H4sIAAAAAAAA/ytITM5OTE/VL4DQelnF+XkMVAYGBgZmJiYK2MRBwNDcSIHB2NTMwNDQzMwAqA7IMDUxA9LUdgg2UFpcklgEdAql5kD8ogCnhwio5lJQUMpLzE1VslJQcihOzi9I1S9JLS7RhSYIJR2QgrLUouLM/DyQGkM9Az1D3YIiqExKanFyUWZBCVQ2BKhVwQVJDKwosbQkI78IJO/tZ+LsbRykxFXLNdA+HwWjYBSMgpENACgAbtAACAAA"
+ integrity := "sha512-yA4FJsVhetynGfOC1jFf79BuS+jrHbm0fhh+aHzCQkOaOBXKf9oBnC4a6DnLLnEsHQDRLYd00cwj8sCXpC+wIg=="
+
+ t.Run("InvalidUpload", func(t *testing.T) {
+ p, err := ParsePackage(bytes.NewReader([]byte{0}))
+ assert.Nil(t, p)
+ assert.Error(t, err)
+ })
+
+ t.Run("InvalidUploadNoData", func(t *testing.T) {
+ b, _ := json.Marshal(packageUpload{})
+ p, err := ParsePackage(bytes.NewReader(b))
+ assert.Nil(t, p)
+ assert.ErrorIs(t, err, ErrInvalidPackage)
+ })
+
+ t.Run("InvalidPackageName", func(t *testing.T) {
+ test := func(t *testing.T, name string) {
+ b, _ := json.Marshal(packageUpload{
+ PackageMetadata: PackageMetadata{
+ ID: name,
+ Name: name,
+ Versions: map[string]*PackageMetadataVersion{
+ packageVersion: {
+ Name: name,
+ },
+ },
+ },
+ })
+
+ p, err := ParsePackage(bytes.NewReader(b))
+ assert.Nil(t, p)
+ assert.ErrorIs(t, err, ErrInvalidPackageName)
+ }
+
+ test(t, " test ")
+ test(t, " test")
+ test(t, "test ")
+ test(t, "te st")
+ test(t, "invalid/scope")
+ test(t, "@invalid/_name")
+ test(t, "@invalid/.name")
+ })
+
+ t.Run("ValidPackageName", func(t *testing.T) {
+ test := func(t *testing.T, name string) {
+ b, _ := json.Marshal(packageUpload{
+ PackageMetadata: PackageMetadata{
+ ID: name,
+ Name: name,
+ Versions: map[string]*PackageMetadataVersion{
+ packageVersion: {
+ Name: name,
+ },
+ },
+ },
+ })
+
+ p, err := ParsePackage(bytes.NewReader(b))
+ assert.Nil(t, p)
+ assert.ErrorIs(t, err, ErrInvalidPackageVersion)
+ }
+
+ test(t, "test")
+ test(t, "@scope/name")
+ test(t, packageFullName)
+ })
+
+ t.Run("InvalidPackageVersion", func(t *testing.T) {
+ version := "first-version"
+ b, _ := json.Marshal(packageUpload{
+ PackageMetadata: PackageMetadata{
+ ID: packageFullName,
+ Name: packageFullName,
+ Versions: map[string]*PackageMetadataVersion{
+ version: {
+ Name: packageFullName,
+ Version: version,
+ },
+ },
+ },
+ })
+
+ p, err := ParsePackage(bytes.NewReader(b))
+ assert.Nil(t, p)
+ assert.ErrorIs(t, err, ErrInvalidPackageVersion)
+ })
+
+ t.Run("InvalidAttachment", func(t *testing.T) {
+ b, _ := json.Marshal(packageUpload{
+ PackageMetadata: PackageMetadata{
+ ID: packageFullName,
+ Name: packageFullName,
+ Versions: map[string]*PackageMetadataVersion{
+ packageVersion: {
+ Name: packageFullName,
+ Version: packageVersion,
+ },
+ },
+ },
+ Attachments: map[string]*PackageAttachment{
+ "dummy.tgz": {},
+ },
+ })
+
+ p, err := ParsePackage(bytes.NewReader(b))
+ assert.Nil(t, p)
+ assert.ErrorIs(t, err, ErrInvalidAttachment)
+ })
+
+ t.Run("InvalidData", func(t *testing.T) {
+ filename := fmt.Sprintf("%s-%s.tgz", packageFullName, packageVersion)
+ b, _ := json.Marshal(packageUpload{
+ PackageMetadata: PackageMetadata{
+ ID: packageFullName,
+ Name: packageFullName,
+ Versions: map[string]*PackageMetadataVersion{
+ packageVersion: {
+ Name: packageFullName,
+ Version: packageVersion,
+ },
+ },
+ },
+ Attachments: map[string]*PackageAttachment{
+ filename: {
+ Data: "/",
+ },
+ },
+ })
+
+ p, err := ParsePackage(bytes.NewReader(b))
+ assert.Nil(t, p)
+ assert.ErrorIs(t, err, ErrInvalidAttachment)
+ })
+
+ t.Run("InvalidIntegrity", func(t *testing.T) {
+ filename := fmt.Sprintf("%s-%s.tgz", packageFullName, packageVersion)
+ b, _ := json.Marshal(packageUpload{
+ PackageMetadata: PackageMetadata{
+ ID: packageFullName,
+ Name: packageFullName,
+ Versions: map[string]*PackageMetadataVersion{
+ packageVersion: {
+ Name: packageFullName,
+ Version: packageVersion,
+ Dist: PackageDistribution{
+ Integrity: "sha512-test==",
+ },
+ },
+ },
+ },
+ Attachments: map[string]*PackageAttachment{
+ filename: {
+ Data: data,
+ },
+ },
+ })
+
+ p, err := ParsePackage(bytes.NewReader(b))
+ assert.Nil(t, p)
+ assert.ErrorIs(t, err, ErrInvalidIntegrity)
+ })
+
+ t.Run("InvalidIntegrity2", func(t *testing.T) {
+ filename := fmt.Sprintf("%s-%s.tgz", packageFullName, packageVersion)
+ b, _ := json.Marshal(packageUpload{
+ PackageMetadata: PackageMetadata{
+ ID: packageFullName,
+ Name: packageFullName,
+ Versions: map[string]*PackageMetadataVersion{
+ packageVersion: {
+ Name: packageFullName,
+ Version: packageVersion,
+ Dist: PackageDistribution{
+ Integrity: integrity,
+ },
+ },
+ },
+ },
+ Attachments: map[string]*PackageAttachment{
+ filename: {
+ Data: base64.StdEncoding.EncodeToString([]byte("data")),
+ },
+ },
+ })
+
+ p, err := ParsePackage(bytes.NewReader(b))
+ assert.Nil(t, p)
+ assert.ErrorIs(t, err, ErrInvalidIntegrity)
+ })
+
+ t.Run("Valid", func(t *testing.T) {
+ filename := fmt.Sprintf("%s-%s.tgz", packageFullName, packageVersion)
+ b, _ := json.Marshal(packageUpload{
+ PackageMetadata: PackageMetadata{
+ ID: packageFullName,
+ Name: packageFullName,
+ DistTags: map[string]string{
+ packageTag: packageVersion,
+ },
+ Versions: map[string]*PackageMetadataVersion{
+ packageVersion: {
+ Name: packageFullName,
+ Version: packageVersion,
+ Description: packageDescription,
+ Author: User{Name: packageAuthor},
+ License: "MIT",
+ Homepage: "https://gitea.io/",
+ Readme: packageDescription,
+ Dependencies: map[string]string{
+ "package": "1.2.0",
+ },
+ Dist: PackageDistribution{
+ Integrity: integrity,
+ },
+ },
+ },
+ },
+ Attachments: map[string]*PackageAttachment{
+ filename: {
+ Data: data,
+ },
+ },
+ })
+
+ p, err := ParsePackage(bytes.NewReader(b))
+ assert.NotNil(t, p)
+ assert.NoError(t, err)
+
+ assert.Equal(t, packageFullName, p.Name)
+ assert.Equal(t, packageVersion, p.Version)
+ assert.Equal(t, []string{packageTag}, p.DistTags)
+ assert.Equal(t, fmt.Sprintf("%s-%s.tgz", strings.Split(packageFullName, "/")[1], packageVersion), p.Filename)
+ b, _ = base64.StdEncoding.DecodeString(data)
+ assert.Equal(t, b, p.Data)
+ assert.Equal(t, packageName, p.Metadata.Name)
+ assert.Equal(t, packageScope, p.Metadata.Scope)
+ assert.Equal(t, packageDescription, p.Metadata.Description)
+ assert.Equal(t, packageDescription, p.Metadata.Readme)
+ assert.Equal(t, packageAuthor, p.Metadata.Author)
+ assert.Equal(t, "MIT", p.Metadata.License)
+ assert.Equal(t, "https://gitea.io/", p.Metadata.ProjectURL)
+ assert.Contains(t, p.Metadata.Dependencies, "package")
+ assert.Equal(t, "1.2.0", p.Metadata.Dependencies["package"])
+ })
+}
diff --git a/modules/packages/npm/metadata.go b/modules/packages/npm/metadata.go
new file mode 100644
index 0000000000..643a4d344b
--- /dev/null
+++ b/modules/packages/npm/metadata.go
@@ -0,0 +1,24 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package npm
+
+// TagProperty is the name of the property for tag management
+const TagProperty = "npm.tag"
+
+// Metadata represents the metadata of a npm package
+type Metadata struct {
+ Scope string `json:"scope,omitempty"`
+ Name string `json:"name,omitempty"`
+ Description string `json:"description,omitempty"`
+ Author string `json:"author,omitempty"`
+ License string `json:"license,omitempty"`
+ ProjectURL string `json:"project_url,omitempty"`
+ Keywords []string `json:"keywords,omitempty"`
+ Dependencies map[string]string `json:"dependencies,omitempty"`
+ DevelopmentDependencies map[string]string `json:"development_dependencies,omitempty"`
+ PeerDependencies map[string]string `json:"peer_dependencies,omitempty"`
+ OptionalDependencies map[string]string `json:"optional_dependencies,omitempty"`
+ Readme string `json:"readme,omitempty"`
+}
diff --git a/modules/packages/nuget/metadata.go b/modules/packages/nuget/metadata.go
new file mode 100644
index 0000000000..797bff45ac
--- /dev/null
+++ b/modules/packages/nuget/metadata.go
@@ -0,0 +1,187 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package nuget
+
+import (
+ "archive/zip"
+ "encoding/xml"
+ "errors"
+ "io"
+ "path/filepath"
+ "regexp"
+ "strings"
+
+ "code.gitea.io/gitea/modules/validation"
+
+ "github.com/hashicorp/go-version"
+)
+
+var (
+ // ErrMissingNuspecFile indicates a missing Nuspec file
+ ErrMissingNuspecFile = errors.New("Nuspec file is missing")
+ // ErrNuspecFileTooLarge indicates a Nuspec file which is too large
+ ErrNuspecFileTooLarge = errors.New("Nuspec file is too large")
+ // ErrNuspecInvalidID indicates an invalid id in the Nuspec file
+ ErrNuspecInvalidID = errors.New("Nuspec file contains an invalid id")
+ // ErrNuspecInvalidVersion indicates an invalid version in the Nuspec file
+ ErrNuspecInvalidVersion = errors.New("Nuspec file contains an invalid version")
+)
+
+// PackageType specifies the package type the metadata describes
+type PackageType int
+
+const (
+ // DependencyPackage represents a package (*.nupkg)
+ DependencyPackage PackageType = iota + 1
+ // SymbolsPackage represents a symbol package (*.snupkg)
+ SymbolsPackage
+
+ PropertySymbolID = "nuget.symbol.id"
+)
+
+var idmatch = regexp.MustCompile(`\A\w+(?:[.-]\w+)*\z`)
+
+const maxNuspecFileSize = 3 * 1024 * 1024
+
+// Package represents a Nuget package
+type Package struct {
+ PackageType PackageType
+ ID string
+ Version string
+ Metadata *Metadata
+}
+
+// Metadata represents the metadata of a Nuget package
+type Metadata struct {
+ Description string `json:"description,omitempty"`
+ ReleaseNotes string `json:"release_notes,omitempty"`
+ Authors string `json:"authors,omitempty"`
+ ProjectURL string `json:"project_url,omitempty"`
+ RepositoryURL string `json:"repository_url,omitempty"`
+ Dependencies map[string][]Dependency `json:"dependencies,omitempty"`
+}
+
+// Dependency represents a dependency of a Nuget package
+type Dependency struct {
+ ID string `json:"id"`
+ Version string `json:"version"`
+}
+
+type nuspecPackage struct {
+ Metadata struct {
+ ID string `xml:"id"`
+ Version string `xml:"version"`
+ Authors string `xml:"authors"`
+ RequireLicenseAcceptance bool `xml:"requireLicenseAcceptance"`
+ ProjectURL string `xml:"projectUrl"`
+ Description string `xml:"description"`
+ ReleaseNotes string `xml:"releaseNotes"`
+ PackageTypes struct {
+ PackageType []struct {
+ Name string `xml:"name,attr"`
+ } `xml:"packageType"`
+ } `xml:"packageTypes"`
+ Repository struct {
+ URL string `xml:"url,attr"`
+ } `xml:"repository"`
+ Dependencies struct {
+ Group []struct {
+ TargetFramework string `xml:"targetFramework,attr"`
+ Dependency []struct {
+ ID string `xml:"id,attr"`
+ Version string `xml:"version,attr"`
+ Exclude string `xml:"exclude,attr"`
+ } `xml:"dependency"`
+ } `xml:"group"`
+ } `xml:"dependencies"`
+ } `xml:"metadata"`
+}
+
+// ParsePackageMetaData parses the metadata of a Nuget package file
+func ParsePackageMetaData(r io.ReaderAt, size int64) (*Package, error) {
+ archive, err := zip.NewReader(r, size)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, file := range archive.File {
+ if filepath.Dir(file.Name) != "." {
+ continue
+ }
+ if strings.HasSuffix(strings.ToLower(file.Name), ".nuspec") {
+ if file.UncompressedSize64 > maxNuspecFileSize {
+ return nil, ErrNuspecFileTooLarge
+ }
+ f, err := archive.Open(file.Name)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ return ParseNuspecMetaData(f)
+ }
+ }
+ return nil, ErrMissingNuspecFile
+}
+
+// ParseNuspecMetaData parses a Nuspec file to retrieve the metadata of a Nuget package
+func ParseNuspecMetaData(r io.Reader) (*Package, error) {
+ var p nuspecPackage
+ if err := xml.NewDecoder(r).Decode(&p); err != nil {
+ return nil, err
+ }
+
+ if !idmatch.MatchString(p.Metadata.ID) {
+ return nil, ErrNuspecInvalidID
+ }
+
+ v, err := version.NewSemver(p.Metadata.Version)
+ if err != nil {
+ return nil, ErrNuspecInvalidVersion
+ }
+
+ if !validation.IsValidURL(p.Metadata.ProjectURL) {
+ p.Metadata.ProjectURL = ""
+ }
+
+ packageType := DependencyPackage
+ for _, pt := range p.Metadata.PackageTypes.PackageType {
+ if pt.Name == "SymbolsPackage" {
+ packageType = SymbolsPackage
+ break
+ }
+ }
+
+ m := &Metadata{
+ Description: p.Metadata.Description,
+ ReleaseNotes: p.Metadata.ReleaseNotes,
+ Authors: p.Metadata.Authors,
+ ProjectURL: p.Metadata.ProjectURL,
+ RepositoryURL: p.Metadata.Repository.URL,
+ Dependencies: make(map[string][]Dependency),
+ }
+
+ for _, group := range p.Metadata.Dependencies.Group {
+ deps := make([]Dependency, 0, len(group.Dependency))
+ for _, dep := range group.Dependency {
+ if dep.ID == "" || dep.Version == "" {
+ continue
+ }
+ deps = append(deps, Dependency{
+ ID: dep.ID,
+ Version: dep.Version,
+ })
+ }
+ if len(deps) > 0 {
+ m.Dependencies[group.TargetFramework] = deps
+ }
+ }
+ return &Package{
+ PackageType: packageType,
+ ID: p.Metadata.ID,
+ Version: v.String(),
+ Metadata: m,
+ }, nil
+}
diff --git a/modules/packages/nuget/metadata_test.go b/modules/packages/nuget/metadata_test.go
new file mode 100644
index 0000000000..e8c7773e97
--- /dev/null
+++ b/modules/packages/nuget/metadata_test.go
@@ -0,0 +1,163 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package nuget
+
+import (
+ "archive/zip"
+ "bytes"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+const (
+ id = "System.Gitea"
+ semver = "1.0.1"
+ authors = "Gitea Authors"
+ projectURL = "https://gitea.io"
+ description = "Package Description"
+ releaseNotes = "Package Release Notes"
+ repositoryURL = "https://gitea.io/gitea/gitea"
+ targetFramework = ".NETStandard2.1"
+ dependencyID = "System.Text.Json"
+ dependencyVersion = "5.0.0"
+)
+
+const nuspecContent = `<?xml version="1.0" encoding="utf-8"?>
+<package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd">
+ <metadata>
+ <id>` + id + `</id>
+ <version>` + semver + `</version>
+ <authors>` + authors + `</authors>
+ <requireLicenseAcceptance>true</requireLicenseAcceptance>
+ <projectUrl>` + projectURL + `</projectUrl>
+ <description>` + description + `</description>
+ <releaseNotes>` + releaseNotes + `</releaseNotes>
+ <repository url="` + repositoryURL + `" />
+ <dependencies>
+ <group targetFramework="` + targetFramework + `">
+ <dependency id="` + dependencyID + `" version="` + dependencyVersion + `" exclude="Build,Analyzers" />
+ </group>
+ </dependencies>
+ </metadata>
+</package>`
+
+const symbolsNuspecContent = `<?xml version="1.0" encoding="utf-8"?>
+<package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd">
+ <metadata>
+ <id>` + id + `</id>
+ <version>` + semver + `</version>
+ <description>` + description + `</description>
+ <packageTypes>
+ <packageType name="SymbolsPackage" />
+ </packageTypes>
+ <dependencies>
+ <group targetFramework="` + targetFramework + `" />
+ </dependencies>
+ </metadata>
+</package>`
+
+func TestParsePackageMetaData(t *testing.T) {
+ createArchive := func(name, content string) []byte {
+ var buf bytes.Buffer
+ archive := zip.NewWriter(&buf)
+ w, _ := archive.Create(name)
+ w.Write([]byte(content))
+ archive.Close()
+ return buf.Bytes()
+ }
+
+ t.Run("MissingNuspecFile", func(t *testing.T) {
+ data := createArchive("dummy.txt", "")
+
+ np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data)))
+ assert.Nil(t, np)
+ assert.ErrorIs(t, err, ErrMissingNuspecFile)
+ })
+
+ t.Run("MissingNuspecFileInRoot", func(t *testing.T) {
+ data := createArchive("sub/package.nuspec", "")
+
+ np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data)))
+ assert.Nil(t, np)
+ assert.ErrorIs(t, err, ErrMissingNuspecFile)
+ })
+
+ t.Run("InvalidNuspecFile", func(t *testing.T) {
+ data := createArchive("package.nuspec", "")
+
+ np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data)))
+ assert.Nil(t, np)
+ assert.Error(t, err)
+ })
+
+ t.Run("InvalidPackageId", func(t *testing.T) {
+ data := createArchive("package.nuspec", `<?xml version="1.0" encoding="utf-8"?>
+ <package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd">
+ <metadata></metadata>
+ </package>`)
+
+ np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data)))
+ assert.Nil(t, np)
+ assert.ErrorIs(t, err, ErrNuspecInvalidID)
+ })
+
+ t.Run("InvalidPackageVersion", func(t *testing.T) {
+ data := createArchive("package.nuspec", `<?xml version="1.0" encoding="utf-8"?>
+ <package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd">
+ <metadata>
+ <id>`+id+`</id>
+ </metadata>
+ </package>`)
+
+ np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data)))
+ assert.Nil(t, np)
+ assert.ErrorIs(t, err, ErrNuspecInvalidVersion)
+ })
+
+ t.Run("Valid", func(t *testing.T) {
+ data := createArchive("package.nuspec", nuspecContent)
+
+ np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data)))
+ assert.NoError(t, err)
+ assert.NotNil(t, np)
+ })
+}
+
+func TestParseNuspecMetaData(t *testing.T) {
+ t.Run("Dependency Package", func(t *testing.T) {
+ np, err := ParseNuspecMetaData(strings.NewReader(nuspecContent))
+ assert.NoError(t, err)
+ assert.NotNil(t, np)
+ assert.Equal(t, DependencyPackage, np.PackageType)
+
+ assert.Equal(t, id, np.ID)
+ assert.Equal(t, semver, np.Version)
+ assert.Equal(t, authors, np.Metadata.Authors)
+ assert.Equal(t, projectURL, np.Metadata.ProjectURL)
+ assert.Equal(t, description, np.Metadata.Description)
+ assert.Equal(t, releaseNotes, np.Metadata.ReleaseNotes)
+ assert.Equal(t, repositoryURL, np.Metadata.RepositoryURL)
+ assert.Len(t, np.Metadata.Dependencies, 1)
+ assert.Contains(t, np.Metadata.Dependencies, targetFramework)
+ deps := np.Metadata.Dependencies[targetFramework]
+ assert.Len(t, deps, 1)
+ assert.Equal(t, dependencyID, deps[0].ID)
+ assert.Equal(t, dependencyVersion, deps[0].Version)
+ })
+
+ t.Run("Symbols Package", func(t *testing.T) {
+ np, err := ParseNuspecMetaData(strings.NewReader(symbolsNuspecContent))
+ assert.NoError(t, err)
+ assert.NotNil(t, np)
+ assert.Equal(t, SymbolsPackage, np.PackageType)
+
+ assert.Equal(t, id, np.ID)
+ assert.Equal(t, semver, np.Version)
+ assert.Equal(t, description, np.Metadata.Description)
+ assert.Empty(t, np.Metadata.Dependencies)
+ })
+}
diff --git a/modules/packages/nuget/symbol_extractor.go b/modules/packages/nuget/symbol_extractor.go
new file mode 100644
index 0000000000..13641ca6ef
--- /dev/null
+++ b/modules/packages/nuget/symbol_extractor.go
@@ -0,0 +1,187 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package nuget
+
+import (
+ "archive/zip"
+ "bytes"
+ "encoding/binary"
+ "errors"
+ "fmt"
+ "io"
+ "path"
+ "path/filepath"
+ "strings"
+
+ "code.gitea.io/gitea/modules/packages"
+)
+
+var (
+ ErrMissingPdbFiles = errors.New("Package does not contain PDB files")
+ ErrInvalidFiles = errors.New("Package contains invalid files")
+ ErrInvalidPdbMagicNumber = errors.New("Invalid Portable PDB magic number")
+ ErrMissingPdbStream = errors.New("Missing PDB stream")
+)
+
+type PortablePdb struct {
+ Name string
+ ID string
+ Content *packages.HashedBuffer
+}
+
+type PortablePdbList []*PortablePdb
+
+func (l PortablePdbList) Close() {
+ for _, pdb := range l {
+ pdb.Content.Close()
+ }
+}
+
+// ExtractPortablePdb extracts PDB files from a .snupkg file
+func ExtractPortablePdb(r io.ReaderAt, size int64) (PortablePdbList, error) {
+ archive, err := zip.NewReader(r, size)
+ if err != nil {
+ return nil, err
+ }
+
+ var pdbs PortablePdbList
+
+ err = func() error {
+ for _, file := range archive.File {
+ if strings.HasSuffix(file.Name, "/") {
+ continue
+ }
+ ext := strings.ToLower(filepath.Ext(file.Name))
+
+ switch ext {
+ case ".nuspec", ".xml", ".psmdcp", ".rels", ".p7s":
+ continue
+ case ".pdb":
+ f, err := archive.Open(file.Name)
+ if err != nil {
+ return err
+ }
+
+ buf, err := packages.CreateHashedBufferFromReader(f, 32*1024*1024)
+
+ f.Close()
+
+ if err != nil {
+ return err
+ }
+
+ id, err := ParseDebugHeaderID(buf)
+ if err != nil {
+ buf.Close()
+ return fmt.Errorf("Invalid PDB file: %v", err)
+ }
+
+ if _, err := buf.Seek(0, io.SeekStart); err != nil {
+ buf.Close()
+ return err
+ }
+
+ pdbs = append(pdbs, &PortablePdb{
+ Name: path.Base(file.Name),
+ ID: id,
+ Content: buf,
+ })
+ default:
+ return ErrInvalidFiles
+ }
+ }
+ return nil
+ }()
+ if err != nil {
+ pdbs.Close()
+ return nil, err
+ }
+
+ if len(pdbs) == 0 {
+ return nil, ErrMissingPdbFiles
+ }
+
+ return pdbs, nil
+}
+
+// ParseDebugHeaderID TODO
+func ParseDebugHeaderID(r io.ReadSeeker) (string, error) {
+ var magic uint32
+ if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {
+ return "", err
+ }
+ if magic != 0x424A5342 {
+ return "", ErrInvalidPdbMagicNumber
+ }
+
+ if _, err := r.Seek(8, io.SeekCurrent); err != nil {
+ return "", err
+ }
+
+ var versionStringSize int32
+ if err := binary.Read(r, binary.LittleEndian, &versionStringSize); err != nil {
+ return "", err
+ }
+ if _, err := r.Seek(int64(versionStringSize), io.SeekCurrent); err != nil {
+ return "", err
+ }
+ if _, err := r.Seek(2, io.SeekCurrent); err != nil {
+ return "", err
+ }
+
+ var streamCount int16
+ if err := binary.Read(r, binary.LittleEndian, &streamCount); err != nil {
+ return "", err
+ }
+
+ read4ByteAlignedString := func(r io.Reader) (string, error) {
+ b := make([]byte, 4)
+ var buf bytes.Buffer
+ for {
+ if _, err := r.Read(b); err != nil {
+ return "", err
+ }
+ if i := bytes.IndexByte(b, 0); i != -1 {
+ buf.Write(b[:i])
+ return buf.String(), nil
+ }
+ buf.Write(b)
+ }
+ }
+
+ for i := 0; i < int(streamCount); i++ {
+ var offset uint32
+ if err := binary.Read(r, binary.LittleEndian, &offset); err != nil {
+ return "", err
+ }
+ if _, err := r.Seek(4, io.SeekCurrent); err != nil {
+ return "", err
+ }
+ name, err := read4ByteAlignedString(r)
+ if err != nil {
+ return "", err
+ }
+
+ if name == "#Pdb" {
+ if _, err := r.Seek(int64(offset), io.SeekStart); err != nil {
+ return "", err
+ }
+
+ b := make([]byte, 16)
+ if _, err := r.Read(b); err != nil {
+ return "", err
+ }
+
+ data1 := binary.LittleEndian.Uint32(b[0:4])
+ data2 := binary.LittleEndian.Uint16(b[4:6])
+ data3 := binary.LittleEndian.Uint16(b[6:8])
+ data4 := b[8:16]
+
+ return fmt.Sprintf("%08x%04x%04x%04x%012x", data1, data2, data3, data4[:2], data4[2:]), nil
+ }
+ }
+
+ return "", ErrMissingPdbStream
+}
diff --git a/modules/packages/nuget/symbol_extractor_test.go b/modules/packages/nuget/symbol_extractor_test.go
new file mode 100644
index 0000000000..892d718caa
--- /dev/null
+++ b/modules/packages/nuget/symbol_extractor_test.go
@@ -0,0 +1,82 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package nuget
+
+import (
+ "archive/zip"
+ "bytes"
+ "encoding/base64"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+const pdbContent = `QlNKQgEAAQAAAAAADAAAAFBEQiB2MS4wAAAAAAAABgB8AAAAWAAAACNQZGIAAAAA1AAAAAgBAAAj
+fgAA3AEAAAQAAAAjU3RyaW5ncwAAAADgAQAABAAAACNVUwDkAQAAMAAAACNHVUlEAAAAFAIAACgB
+AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==`
+
+func TestExtractPortablePdb(t *testing.T) {
+ createArchive := func(name string, content []byte) []byte {
+ var buf bytes.Buffer
+ archive := zip.NewWriter(&buf)
+ w, _ := archive.Create(name)
+ w.Write(content)
+ archive.Close()
+ return buf.Bytes()
+ }
+
+ t.Run("MissingPdbFiles", func(t *testing.T) {
+ var buf bytes.Buffer
+ zip.NewWriter(&buf).Close()
+
+ pdbs, err := ExtractPortablePdb(bytes.NewReader(buf.Bytes()), int64(buf.Len()))
+ assert.ErrorIs(t, err, ErrMissingPdbFiles)
+ assert.Empty(t, pdbs)
+ })
+
+ t.Run("InvalidFiles", func(t *testing.T) {
+ data := createArchive("sub/test.bin", []byte{})
+
+ pdbs, err := ExtractPortablePdb(bytes.NewReader(data), int64(len(data)))
+ assert.ErrorIs(t, err, ErrInvalidFiles)
+ assert.Empty(t, pdbs)
+ })
+
+ t.Run("Valid", func(t *testing.T) {
+ b, _ := base64.StdEncoding.DecodeString(pdbContent)
+ data := createArchive("test.pdb", b)
+
+ pdbs, err := ExtractPortablePdb(bytes.NewReader(data), int64(len(data)))
+ assert.NoError(t, err)
+ assert.Len(t, pdbs, 1)
+ assert.Equal(t, "test.pdb", pdbs[0].Name)
+ assert.Equal(t, "d910bb6948bd4c6cb40155bcf52c3c94", pdbs[0].ID)
+ pdbs.Close()
+ })
+}
+
+func TestParseDebugHeaderID(t *testing.T) {
+ t.Run("InvalidPdbMagicNumber", func(t *testing.T) {
+ id, err := ParseDebugHeaderID(bytes.NewReader([]byte{0, 0, 0, 0}))
+ assert.ErrorIs(t, err, ErrInvalidPdbMagicNumber)
+ assert.Empty(t, id)
+ })
+
+ t.Run("MissingPdbStream", func(t *testing.T) {
+ b, _ := base64.StdEncoding.DecodeString(`QlNKQgEAAQAAAAAADAAAAFBEQiB2MS4wAAAAAAAAAQB8AAAAWAAAACNVUwA=`)
+
+ id, err := ParseDebugHeaderID(bytes.NewReader(b))
+ assert.ErrorIs(t, err, ErrMissingPdbStream)
+ assert.Empty(t, id)
+ })
+
+ t.Run("Valid", func(t *testing.T) {
+ b, _ := base64.StdEncoding.DecodeString(pdbContent)
+
+ id, err := ParseDebugHeaderID(bytes.NewReader(b))
+ assert.NoError(t, err)
+ assert.Equal(t, "d910bb6948bd4c6cb40155bcf52c3c94", id)
+ })
+}
diff --git a/modules/packages/pypi/metadata.go b/modules/packages/pypi/metadata.go
new file mode 100644
index 0000000000..df367d10e2
--- /dev/null
+++ b/modules/packages/pypi/metadata.go
@@ -0,0 +1,16 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package pypi
+
+// Metadata represents the metadata of a PyPI package
+type Metadata struct {
+ Author string `json:"author,omitempty"`
+ Description string `json:"description,omitempty"`
+ LongDescription string `json:"long_description,omitempty"`
+ Summary string `json:"summary,omitempty"`
+ ProjectURL string `json:"project_url,omitempty"`
+ License string `json:"license,omitempty"`
+ RequiresPython string `json:"requires_python,omitempty"`
+}
diff --git a/modules/packages/rubygems/marshal.go b/modules/packages/rubygems/marshal.go
new file mode 100644
index 0000000000..2c45042fa8
--- /dev/null
+++ b/modules/packages/rubygems/marshal.go
@@ -0,0 +1,311 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package rubygems
+
+import (
+ "bufio"
+ "bytes"
+ "errors"
+ "io"
+ "reflect"
+)
+
+const (
+ majorVersion = 4
+ minorVersion = 8
+
+ typeNil = '0'
+ typeTrue = 'T'
+ typeFalse = 'F'
+ typeFixnum = 'i'
+ typeString = '"'
+ typeSymbol = ':'
+ typeSymbolLink = ';'
+ typeArray = '['
+ typeIVar = 'I'
+ typeUserMarshal = 'U'
+ typeUserDef = 'u'
+ typeObject = 'o'
+)
+
+var (
+ // ErrUnsupportedType indicates an unsupported type
+ ErrUnsupportedType = errors.New("Type is unsupported")
+ // ErrInvalidIntRange indicates an invalid number range
+ ErrInvalidIntRange = errors.New("Number is not in valid range")
+)
+
+// RubyUserMarshal is a Ruby object that has a marshal_load function.
+type RubyUserMarshal struct {
+ Name string
+ Value interface{}
+}
+
+// RubyUserDef is a Ruby object that has a _load function.
+type RubyUserDef struct {
+ Name string
+ Value interface{}
+}
+
+// RubyObject is a default Ruby object.
+type RubyObject struct {
+ Name string
+ Member map[string]interface{}
+}
+
+// MarshalEncoder mimics Rubys Marshal class.
+// Note: Only supports types used by the RubyGems package registry.
+type MarshalEncoder struct {
+ w *bufio.Writer
+ symbols map[string]int
+}
+
+// NewMarshalEncoder creates a new MarshalEncoder
+func NewMarshalEncoder(w io.Writer) *MarshalEncoder {
+ return &MarshalEncoder{
+ w: bufio.NewWriter(w),
+ symbols: map[string]int{},
+ }
+}
+
+// Encode encodes the given type
+func (e *MarshalEncoder) Encode(v interface{}) error {
+ if _, err := e.w.Write([]byte{majorVersion, minorVersion}); err != nil {
+ return err
+ }
+
+ if err := e.marshal(v); err != nil {
+ return err
+ }
+
+ return e.w.Flush()
+}
+
+func (e *MarshalEncoder) marshal(v interface{}) error {
+ if v == nil {
+ return e.marshalNil()
+ }
+
+ val := reflect.ValueOf(v)
+ typ := reflect.TypeOf(v)
+
+ if typ.Kind() == reflect.Ptr {
+ val = val.Elem()
+ typ = typ.Elem()
+ }
+
+ switch typ.Kind() {
+ case reflect.Bool:
+ return e.marshalBool(val.Bool())
+ case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32:
+ return e.marshalInt(val.Int())
+ case reflect.String:
+ return e.marshalString(val.String())
+ case reflect.Slice, reflect.Array:
+ return e.marshalArray(val)
+ }
+
+ switch typ.Name() {
+ case "RubyUserMarshal":
+ return e.marshalUserMarshal(val.Interface().(RubyUserMarshal))
+ case "RubyUserDef":
+ return e.marshalUserDef(val.Interface().(RubyUserDef))
+ case "RubyObject":
+ return e.marshalObject(val.Interface().(RubyObject))
+ }
+
+ return ErrUnsupportedType
+}
+
+func (e *MarshalEncoder) marshalNil() error {
+ return e.w.WriteByte(typeNil)
+}
+
+func (e *MarshalEncoder) marshalBool(b bool) error {
+ if b {
+ return e.w.WriteByte(typeTrue)
+ }
+ return e.w.WriteByte(typeFalse)
+}
+
+func (e *MarshalEncoder) marshalInt(i int64) error {
+ if err := e.w.WriteByte(typeFixnum); err != nil {
+ return err
+ }
+
+ return e.marshalIntInternal(i)
+}
+
+func (e *MarshalEncoder) marshalIntInternal(i int64) error {
+ if i == 0 {
+ return e.w.WriteByte(0)
+ } else if 0 < i && i < 123 {
+ return e.w.WriteByte(byte(i + 5))
+ } else if -124 < i && i <= -1 {
+ return e.w.WriteByte(byte(i - 5))
+ }
+
+ var len int
+ if 122 < i && i <= 0xff {
+ len = 1
+ } else if 0xff < i && i <= 0xffff {
+ len = 2
+ } else if 0xffff < i && i <= 0xffffff {
+ len = 3
+ } else if 0xffffff < i && i <= 0x3fffffff {
+ len = 4
+ } else if -0x100 <= i && i < -123 {
+ len = -1
+ } else if -0x10000 <= i && i < -0x100 {
+ len = -2
+ } else if -0x1000000 <= i && i < -0x100000 {
+ len = -3
+ } else if -0x40000000 <= i && i < -0x1000000 {
+ len = -4
+ } else {
+ return ErrInvalidIntRange
+ }
+
+ if err := e.w.WriteByte(byte(len)); err != nil {
+ return err
+ }
+ if len < 0 {
+ len = -len
+ }
+
+ for c := 0; c < len; c++ {
+ if err := e.w.WriteByte(byte(i >> uint(8*c) & 0xff)); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (e *MarshalEncoder) marshalString(str string) error {
+ if err := e.w.WriteByte(typeIVar); err != nil {
+ return err
+ }
+
+ if err := e.marshalRawString(str); err != nil {
+ return err
+ }
+
+ if err := e.marshalIntInternal(1); err != nil {
+ return err
+ }
+
+ if err := e.marshalSymbol("E"); err != nil {
+ return err
+ }
+
+ return e.marshalBool(true)
+}
+
+func (e *MarshalEncoder) marshalRawString(str string) error {
+ if err := e.w.WriteByte(typeString); err != nil {
+ return err
+ }
+
+ if err := e.marshalIntInternal(int64(len(str))); err != nil {
+ return err
+ }
+
+ _, err := e.w.WriteString(str)
+ return err
+}
+
+func (e *MarshalEncoder) marshalSymbol(str string) error {
+ if index, ok := e.symbols[str]; ok {
+ if err := e.w.WriteByte(typeSymbolLink); err != nil {
+ return err
+ }
+ return e.marshalIntInternal(int64(index))
+ }
+
+ e.symbols[str] = len(e.symbols)
+
+ if err := e.w.WriteByte(typeSymbol); err != nil {
+ return err
+ }
+
+ if err := e.marshalIntInternal(int64(len(str))); err != nil {
+ return err
+ }
+
+ _, err := e.w.WriteString(str)
+ return err
+}
+
+func (e *MarshalEncoder) marshalArray(arr reflect.Value) error {
+ if err := e.w.WriteByte(typeArray); err != nil {
+ return err
+ }
+
+ len := arr.Len()
+
+ if err := e.marshalIntInternal(int64(len)); err != nil {
+ return err
+ }
+
+ for i := 0; i < len; i++ {
+ if err := e.marshal(arr.Index(i).Interface()); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (e *MarshalEncoder) marshalUserMarshal(userMarshal RubyUserMarshal) error {
+ if err := e.w.WriteByte(typeUserMarshal); err != nil {
+ return err
+ }
+
+ if err := e.marshalSymbol(userMarshal.Name); err != nil {
+ return err
+ }
+
+ return e.marshal(userMarshal.Value)
+}
+
+func (e *MarshalEncoder) marshalUserDef(userDef RubyUserDef) error {
+ var buf bytes.Buffer
+ if err := NewMarshalEncoder(&buf).Encode(userDef.Value); err != nil {
+ return err
+ }
+
+ if err := e.w.WriteByte(typeUserDef); err != nil {
+ return err
+ }
+ if err := e.marshalSymbol(userDef.Name); err != nil {
+ return err
+ }
+ if err := e.marshalIntInternal(int64(buf.Len())); err != nil {
+ return err
+ }
+ _, err := e.w.Write(buf.Bytes())
+ return err
+}
+
+func (e *MarshalEncoder) marshalObject(obj RubyObject) error {
+ if err := e.w.WriteByte(typeObject); err != nil {
+ return err
+ }
+ if err := e.marshalSymbol(obj.Name); err != nil {
+ return err
+ }
+ if err := e.marshalIntInternal(int64(len(obj.Member))); err != nil {
+ return err
+ }
+ for k, v := range obj.Member {
+ if err := e.marshalSymbol(k); err != nil {
+ return err
+ }
+ if err := e.marshal(v); err != nil {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/modules/packages/rubygems/marshal_test.go b/modules/packages/rubygems/marshal_test.go
new file mode 100644
index 0000000000..e5963ebcd6
--- /dev/null
+++ b/modules/packages/rubygems/marshal_test.go
@@ -0,0 +1,99 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package rubygems
+
+import (
+ "bytes"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestMinimalEncoder(t *testing.T) {
+ cases := []struct {
+ Value interface{}
+ Expected []byte
+ Error error
+ }{
+ {
+ Value: nil,
+ Expected: []byte{4, 8, 0x30},
+ },
+ {
+ Value: true,
+ Expected: []byte{4, 8, 'T'},
+ },
+ {
+ Value: false,
+ Expected: []byte{4, 8, 'F'},
+ },
+ {
+ Value: 0,
+ Expected: []byte{4, 8, 'i', 0},
+ },
+ {
+ Value: 1,
+ Expected: []byte{4, 8, 'i', 6},
+ },
+ {
+ Value: -1,
+ Expected: []byte{4, 8, 'i', 0xfa},
+ },
+ {
+ Value: 0x1fffffff,
+ Expected: []byte{4, 8, 'i', 4, 0xff, 0xff, 0xff, 0x1f},
+ },
+ {
+ Value: 0x41000000,
+ Error: ErrInvalidIntRange,
+ },
+ {
+ Value: "test",
+ Expected: []byte{4, 8, 'I', '"', 9, 't', 'e', 's', 't', 6, ':', 6, 'E', 'T'},
+ },
+ {
+ Value: []int{1, 2},
+ Expected: []byte{4, 8, '[', 7, 'i', 6, 'i', 7},
+ },
+ {
+ Value: &RubyUserMarshal{
+ Name: "Test",
+ Value: 4,
+ },
+ Expected: []byte{4, 8, 'U', ':', 9, 'T', 'e', 's', 't', 'i', 9},
+ },
+ {
+ Value: &RubyUserDef{
+ Name: "Test",
+ Value: 4,
+ },
+ Expected: []byte{4, 8, 'u', ':', 9, 'T', 'e', 's', 't', 9, 4, 8, 'i', 9},
+ },
+ {
+ Value: &RubyObject{
+ Name: "Test",
+ Member: map[string]interface{}{
+ "test": 4,
+ },
+ },
+ Expected: []byte{4, 8, 'o', ':', 9, 'T', 'e', 's', 't', 6, ':', 9, 't', 'e', 's', 't', 'i', 9},
+ },
+ {
+ Value: &struct {
+ Name string
+ }{
+ "test",
+ },
+ Error: ErrUnsupportedType,
+ },
+ }
+
+ for i, c := range cases {
+ var b bytes.Buffer
+ err := NewMarshalEncoder(&b).Encode(c.Value)
+ assert.ErrorIs(t, err, c.Error)
+ assert.Equal(t, c.Expected, b.Bytes(), "case %d", i)
+ }
+}
diff --git a/modules/packages/rubygems/metadata.go b/modules/packages/rubygems/metadata.go
new file mode 100644
index 0000000000..942f205fc3
--- /dev/null
+++ b/modules/packages/rubygems/metadata.go
@@ -0,0 +1,222 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package rubygems
+
+import (
+ "archive/tar"
+ "compress/gzip"
+ "errors"
+ "io"
+ "regexp"
+ "strings"
+
+ "code.gitea.io/gitea/modules/validation"
+
+ "gopkg.in/yaml.v2"
+)
+
+var (
+ // ErrMissingMetadataFile indicates a missing metadata.gz file
+ ErrMissingMetadataFile = errors.New("Metadata file is missing")
+ // ErrInvalidName indicates an invalid id in the metadata.gz file
+ ErrInvalidName = errors.New("Metadata file contains an invalid name")
+ // ErrInvalidVersion indicates an invalid version in the metadata.gz file
+ ErrInvalidVersion = errors.New("Metadata file contains an invalid version")
+)
+
+var versionMatcher = regexp.MustCompile(`\A[0-9]+(?:\.[0-9a-zA-Z]+)*(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?\z`)
+
+// Package represents a RubyGems package
+type Package struct {
+ Name string
+ Version string
+ Metadata *Metadata
+}
+
+// Metadata represents the metadata of a RubyGems package
+type Metadata struct {
+ Platform string `json:"platform,omitempty"`
+ Description string `json:"description,omitempty"`
+ Summary string `json:"summary,omitempty"`
+ Authors []string `json:"authors,omitempty"`
+ Licenses []string `json:"licenses,omitempty"`
+ RequiredRubyVersion []VersionRequirement `json:"required_ruby_version,omitempty"`
+ RequiredRubygemsVersion []VersionRequirement `json:"required_rubygems_version,omitempty"`
+ ProjectURL string `json:"project_url,omitempty"`
+ RuntimeDependencies []Dependency `json:"runtime_dependencies,omitempty"`
+ DevelopmentDependencies []Dependency `json:"development_dependencies,omitempty"`
+}
+
+// VersionRequirement represents a version restriction
+type VersionRequirement struct {
+ Restriction string `json:"restriction"`
+ Version string `json:"version"`
+}
+
+// Dependency represents a dependency of a RubyGems package
+type Dependency struct {
+ Name string `json:"name"`
+ Version []VersionRequirement `json:"version"`
+}
+
+type gemspec struct {
+ Name string `yaml:"name"`
+ Version struct {
+ Version string `yaml:"version"`
+ } `yaml:"version"`
+ Platform string `yaml:"platform"`
+ Authors []string `yaml:"authors"`
+ Autorequire interface{} `yaml:"autorequire"`
+ Bindir string `yaml:"bindir"`
+ CertChain []interface{} `yaml:"cert_chain"`
+ Date string `yaml:"date"`
+ Dependencies []struct {
+ Name string `yaml:"name"`
+ Requirement requirement `yaml:"requirement"`
+ Type string `yaml:"type"`
+ Prerelease bool `yaml:"prerelease"`
+ VersionRequirements requirement `yaml:"version_requirements"`
+ } `yaml:"dependencies"`
+ Description string `yaml:"description"`
+ Email string `yaml:"email"`
+ Executables []string `yaml:"executables"`
+ Extensions []interface{} `yaml:"extensions"`
+ ExtraRdocFiles []string `yaml:"extra_rdoc_files"`
+ Files []string `yaml:"files"`
+ Homepage string `yaml:"homepage"`
+ Licenses []string `yaml:"licenses"`
+ Metadata struct {
+ BugTrackerURI string `yaml:"bug_tracker_uri"`
+ ChangelogURI string `yaml:"changelog_uri"`
+ DocumentationURI string `yaml:"documentation_uri"`
+ SourceCodeURI string `yaml:"source_code_uri"`
+ } `yaml:"metadata"`
+ PostInstallMessage interface{} `yaml:"post_install_message"`
+ RdocOptions []interface{} `yaml:"rdoc_options"`
+ RequirePaths []string `yaml:"require_paths"`
+ RequiredRubyVersion requirement `yaml:"required_ruby_version"`
+ RequiredRubygemsVersion requirement `yaml:"required_rubygems_version"`
+ Requirements []interface{} `yaml:"requirements"`
+ RubygemsVersion string `yaml:"rubygems_version"`
+ SigningKey interface{} `yaml:"signing_key"`
+ SpecificationVersion int `yaml:"specification_version"`
+ Summary string `yaml:"summary"`
+ TestFiles []interface{} `yaml:"test_files"`
+}
+
+type requirement struct {
+ Requirements [][]interface{} `yaml:"requirements"`
+}
+
+// AsVersionRequirement converts into []VersionRequirement
+func (r requirement) AsVersionRequirement() []VersionRequirement {
+ requirements := make([]VersionRequirement, 0, len(r.Requirements))
+ for _, req := range r.Requirements {
+ if len(req) != 2 {
+ continue
+ }
+ restriction, ok := req[0].(string)
+ if !ok {
+ continue
+ }
+ vm, ok := req[1].(map[interface{}]interface{})
+ if !ok {
+ continue
+ }
+ versionInt, ok := vm["version"]
+ if !ok {
+ continue
+ }
+ version, ok := versionInt.(string)
+ if !ok || version == "0" {
+ continue
+ }
+
+ requirements = append(requirements, VersionRequirement{
+ Restriction: restriction,
+ Version: version,
+ })
+ }
+ return requirements
+}
+
+// ParsePackageMetaData parses the metadata of a Gem package file
+func ParsePackageMetaData(r io.Reader) (*Package, error) {
+ archive := tar.NewReader(r)
+ for {
+ hdr, err := archive.Next()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ if hdr.Name == "metadata.gz" {
+ return parseMetadataFile(archive)
+ }
+ }
+
+ return nil, ErrMissingMetadataFile
+}
+
+func parseMetadataFile(r io.Reader) (*Package, error) {
+ zr, err := gzip.NewReader(r)
+ if err != nil {
+ return nil, err
+ }
+ defer zr.Close()
+
+ var spec gemspec
+ if err := yaml.NewDecoder(zr).Decode(&spec); err != nil {
+ return nil, err
+ }
+
+ if len(spec.Name) == 0 || strings.Contains(spec.Name, "/") {
+ return nil, ErrInvalidName
+ }
+
+ if !versionMatcher.MatchString(spec.Version.Version) {
+ return nil, ErrInvalidVersion
+ }
+
+ if !validation.IsValidURL(spec.Homepage) {
+ spec.Homepage = ""
+ }
+ if !validation.IsValidURL(spec.Metadata.SourceCodeURI) {
+ spec.Metadata.SourceCodeURI = ""
+ }
+
+ m := &Metadata{
+ Platform: spec.Platform,
+ Description: spec.Description,
+ Summary: spec.Summary,
+ Authors: spec.Authors,
+ Licenses: spec.Licenses,
+ ProjectURL: spec.Homepage,
+ RequiredRubyVersion: spec.RequiredRubyVersion.AsVersionRequirement(),
+ RequiredRubygemsVersion: spec.RequiredRubygemsVersion.AsVersionRequirement(),
+ DevelopmentDependencies: make([]Dependency, 0, 5),
+ RuntimeDependencies: make([]Dependency, 0, 5),
+ }
+
+ for _, gemdep := range spec.Dependencies {
+ dep := Dependency{
+ Name: gemdep.Name,
+ Version: gemdep.Requirement.AsVersionRequirement(),
+ }
+ if gemdep.Type == ":runtime" {
+ m.RuntimeDependencies = append(m.RuntimeDependencies, dep)
+ } else {
+ m.DevelopmentDependencies = append(m.DevelopmentDependencies, dep)
+ }
+ }
+
+ return &Package{
+ Name: spec.Name,
+ Version: spec.Version.Version,
+ Metadata: m,
+ }, nil
+}
diff --git a/modules/packages/rubygems/metadata_test.go b/modules/packages/rubygems/metadata_test.go
new file mode 100644
index 0000000000..dbefa9c236
--- /dev/null
+++ b/modules/packages/rubygems/metadata_test.go
@@ -0,0 +1,89 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package rubygems
+
+import (
+ "archive/tar"
+ "bytes"
+ "encoding/base64"
+ "io"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestParsePackageMetaData(t *testing.T) {
+ createArchive := func(filename string, content []byte) io.Reader {
+ var buf bytes.Buffer
+ tw := tar.NewWriter(&buf)
+ hdr := &tar.Header{
+ Name: filename,
+ Mode: 0o600,
+ Size: int64(len(content)),
+ }
+ tw.WriteHeader(hdr)
+ tw.Write(content)
+ tw.Close()
+ return &buf
+ }
+
+ t.Run("MissingMetadataFile", func(t *testing.T) {
+ data := createArchive("dummy.txt", []byte{0})
+
+ rp, err := ParsePackageMetaData(data)
+ assert.ErrorIs(t, err, ErrMissingMetadataFile)
+ assert.Nil(t, rp)
+ })
+
+ t.Run("Valid", func(t *testing.T) {
+ content, _ := base64.StdEncoding.DecodeString("H4sICHC/I2EEAG1ldGFkYXRhAAEeAOH/bmFtZTogZwp2ZXJzaW9uOgogIHZlcnNpb246IDEKWw35Tx4AAAA=")
+ data := createArchive("metadata.gz", content)
+
+ rp, err := ParsePackageMetaData(data)
+ assert.NoError(t, err)
+ assert.NotNil(t, rp)
+ })
+}
+
+func TestParseMetadataFile(t *testing.T) {
+ content, _ := base64.StdEncoding.DecodeString(`H4sIAMe7I2ECA9VVTW/UMBC9+1eYXvaUbJpSQBZUHJAqDlwK4kCFIseZzZrGH9iTqisEv52Js9nd
+0KqggiqRXWnX45n3ZuZ5nCzL+JPQ15ulq7+AQnEORoj3HpReaSVRO8usNCB4qxEku4YQySbuCPo4
+bjHOd07HeZGfMt9JXLlgBB9imOxx7UIULOPnCZMMLsDXXgeiYbW2jQ6C0y9TELBSa6kJ6/IzaySS
+R1mUx1nxIitPeFGI9M2L6eGfWAMebANWaUgktzN9M3lsKNmxutBb1AYyCibbNhsDFu+q9GK/Tc4z
+d2IcLBl9js5eHaXFsLyvXeNz0LQyL/YoLx8EsiCMBZlx46k6sS2PDD5AgA5kJPNKdhH2elWzOv7n
+uv9Q9Aau/6ngP84elvNpXh5oRVlB5/yW7BH0+qu0G4gqaI/JdEHBFBS5l+pKtsARIjIwUnfj8Le0
++TrdJLl2DG5A9SjrjgZ1mG+4QbAD+G4ZZBUap6qVnnzGf6Rwp+vliBRqtnYGPBEKvkb0USyXE8mS
+dVoR6hj07u0HZgAl3SRS8G/fmXcRK20jyq6rDMSYQFgidamqkXbbuspLXE/0k7GphtKqe67GuRC/
+yjAbmt9LsOMp8xMamFkSQ38fP5EFjdz8LA4do2C69VvqWXAJgrPbKZb58/xZXrKoW6ttW13Bhvzi
+4ftn7/yUxd4YGcglvTmmY8aGY3ZwRn4CqcWcidUGAAA=`)
+ rp, err := parseMetadataFile(bytes.NewReader(content))
+ assert.NoError(t, err)
+ assert.NotNil(t, rp)
+
+ assert.Equal(t, "gitea", rp.Name)
+ assert.Equal(t, "1.0.5", rp.Version)
+ assert.Equal(t, "ruby", rp.Metadata.Platform)
+ assert.Equal(t, "Gitea package", rp.Metadata.Summary)
+ assert.Equal(t, "RubyGems package test", rp.Metadata.Description)
+ assert.Equal(t, []string{"Gitea"}, rp.Metadata.Authors)
+ assert.Equal(t, "https://gitea.io/", rp.Metadata.ProjectURL)
+ assert.Equal(t, []string{"MIT"}, rp.Metadata.Licenses)
+ assert.Empty(t, rp.Metadata.RequiredRubygemsVersion)
+ assert.Len(t, rp.Metadata.RequiredRubyVersion, 1)
+ assert.Equal(t, ">=", rp.Metadata.RequiredRubyVersion[0].Restriction)
+ assert.Equal(t, "2.3.0", rp.Metadata.RequiredRubyVersion[0].Version)
+ assert.Len(t, rp.Metadata.RuntimeDependencies, 1)
+ assert.Equal(t, "runtime-dep", rp.Metadata.RuntimeDependencies[0].Name)
+ assert.Len(t, rp.Metadata.RuntimeDependencies[0].Version, 2)
+ assert.Equal(t, ">=", rp.Metadata.RuntimeDependencies[0].Version[0].Restriction)
+ assert.Equal(t, "1.2.0", rp.Metadata.RuntimeDependencies[0].Version[0].Version)
+ assert.Equal(t, "<", rp.Metadata.RuntimeDependencies[0].Version[1].Restriction)
+ assert.Equal(t, "2.0", rp.Metadata.RuntimeDependencies[0].Version[1].Version)
+ assert.Len(t, rp.Metadata.DevelopmentDependencies, 1)
+ assert.Equal(t, "dev-dep", rp.Metadata.DevelopmentDependencies[0].Name)
+ assert.Len(t, rp.Metadata.DevelopmentDependencies[0].Version, 1)
+ assert.Equal(t, "~>", rp.Metadata.DevelopmentDependencies[0].Version[0].Restriction)
+ assert.Equal(t, "5.2", rp.Metadata.DevelopmentDependencies[0].Version[0].Version)
+}
diff --git a/modules/setting/packages.go b/modules/setting/packages.go
new file mode 100644
index 0000000000..65653b990e
--- /dev/null
+++ b/modules/setting/packages.go
@@ -0,0 +1,47 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package setting
+
+import (
+ "os"
+ "path/filepath"
+
+ "code.gitea.io/gitea/modules/log"
+)
+
+// Package registry settings
+var (
+ Packages = struct {
+ Storage
+ Enabled bool
+ ChunkedUploadPath string
+ RegistryHost string
+ }{
+ Enabled: true,
+ }
+)
+
+func newPackages() {
+ sec := Cfg.Section("packages")
+ if err := sec.MapTo(&Packages); err != nil {
+ log.Fatal("Failed to map Packages settings: %v", err)
+ }
+
+ Packages.Storage = getStorage("packages", "", nil)
+
+ Packages.RegistryHost = Domain
+ if (Protocol == HTTP && HTTPPort != "80") || (Protocol == HTTPS && HTTPPort != "443") {
+ Packages.RegistryHost += ":" + HTTPPort
+ }
+
+ Packages.ChunkedUploadPath = filepath.ToSlash(sec.Key("CHUNKED_UPLOAD_PATH").MustString("tmp/package-upload"))
+ if !filepath.IsAbs(Packages.ChunkedUploadPath) {
+ Packages.ChunkedUploadPath = filepath.ToSlash(filepath.Join(AppDataPath, Packages.ChunkedUploadPath))
+ }
+
+ if err := os.MkdirAll(Packages.ChunkedUploadPath, os.ModePerm); err != nil {
+ log.Error("Unable to create chunked upload directory: %s (%v)", Packages.ChunkedUploadPath, err)
+ }
+}
diff --git a/modules/setting/setting.go b/modules/setting/setting.go
index c80fc3d204..17a02bf5a1 100644
--- a/modules/setting/setting.go
+++ b/modules/setting/setting.go
@@ -212,6 +212,7 @@ var (
MembersPagingNum int
FeedMaxCommitNum int
FeedPagingNum int
+ PackagesPagingNum int
GraphMaxCommitNum int
CodeCommentLines int
ReactionMaxUserNum int
@@ -264,6 +265,7 @@ var (
MembersPagingNum: 20,
FeedMaxCommitNum: 5,
FeedPagingNum: 20,
+ PackagesPagingNum: 20,
GraphMaxCommitNum: 100,
CodeCommentLines: 4,
ReactionMaxUserNum: 10,
@@ -1016,6 +1018,8 @@ func loadFromConf(allowEmpty bool, extraConfig string) {
newPictureService()
+ newPackages()
+
if err = Cfg.Section("ui").MapTo(&UI); err != nil {
log.Fatal("Failed to map UI settings: %v", err)
} else if err = Cfg.Section("markdown").MapTo(&Markdown); err != nil {
diff --git a/modules/storage/storage.go b/modules/storage/storage.go
index f11e1ac743..ef7f6029a5 100644
--- a/modules/storage/storage.go
+++ b/modules/storage/storage.go
@@ -123,6 +123,9 @@ var (
// RepoArchives represents repository archives storage
RepoArchives ObjectStorage
+
+ // Packages represents packages storage
+ Packages ObjectStorage
)
// Init init the stoarge
@@ -143,7 +146,11 @@ func Init() error {
return err
}
- return initRepoArchives()
+ if err := initRepoArchives(); err != nil {
+ return err
+ }
+
+ return initPackages()
}
// NewStorage takes a storage type and some config and returns an ObjectStorage or an error
@@ -188,3 +195,9 @@ func initRepoArchives() (err error) {
RepoArchives, err = NewStorage(setting.RepoArchive.Storage.Type, &setting.RepoArchive.Storage)
return
}
+
+func initPackages() (err error) {
+ log.Info("Initialising Packages storage with type: %s", setting.Packages.Storage.Type)
+ Packages, err = NewStorage(setting.Packages.Storage.Type, &setting.Packages.Storage)
+ return
+}
diff --git a/modules/structs/hook.go b/modules/structs/hook.go
index e4d7652c72..07d51915de 100644
--- a/modules/structs/hook.go
+++ b/modules/structs/hook.go
@@ -110,6 +110,7 @@ var (
_ Payloader = &PullRequestPayload{}
_ Payloader = &RepositoryPayload{}
_ Payloader = &ReleasePayload{}
+ _ Payloader = &PackagePayload{}
)
// _________ __
@@ -425,3 +426,27 @@ type RepositoryPayload struct {
func (p *RepositoryPayload) JSONPayload() ([]byte, error) {
return json.MarshalIndent(p, "", " ")
}
+
+// HookPackageAction an action that happens to a package
+type HookPackageAction string
+
+const (
+ // HookPackageCreated created
+ HookPackageCreated HookPackageAction = "created"
+ // HookPackageDeleted deleted
+ HookPackageDeleted HookPackageAction = "deleted"
+)
+
+// PackagePayload represents a package payload
+type PackagePayload struct {
+ Action HookPackageAction `json:"action"`
+ Repository *Repository `json:"repository"`
+ Package *Package `json:"package"`
+ Organization *User `json:"organization"`
+ Sender *User `json:"sender"`
+}
+
+// JSONPayload implements Payload
+func (p *PackagePayload) JSONPayload() ([]byte, error) {
+ return json.MarshalIndent(p, "", " ")
+}
diff --git a/modules/structs/package.go b/modules/structs/package.go
new file mode 100644
index 0000000000..fbdd6c90aa
--- /dev/null
+++ b/modules/structs/package.go
@@ -0,0 +1,33 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package structs
+
+import (
+ "time"
+)
+
+// Package represents a package
+type Package struct {
+ ID int64 `json:"id"`
+ Owner *User `json:"owner"`
+ Repository *Repository `json:"repository"`
+ Creator *User `json:"creator"`
+ Type string `json:"type"`
+ Name string `json:"name"`
+ Version string `json:"version"`
+ // swagger:strfmt date-time
+ CreatedAt time.Time `json:"created_at"`
+}
+
+// PackageFile represents a package file
+type PackageFile struct {
+ ID int64 `json:"id"`
+ Size int64
+ Name string `json:"name"`
+ HashMD5 string `json:"md5"`
+ HashSHA1 string `json:"sha1"`
+ HashSHA256 string `json:"sha256"`
+ HashSHA512 string `json:"sha512"`
+}
diff --git a/modules/templates/helper.go b/modules/templates/helper.go
index 80ad7066a7..1201710b92 100644
--- a/modules/templates/helper.go
+++ b/modules/templates/helper.go
@@ -34,6 +34,7 @@ import (
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/markup"
+ "code.gitea.io/gitea/modules/markup/markdown"
"code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/svg"
@@ -161,7 +162,16 @@ func NewFuncMap() []template.FuncMap {
"RenderEmojiPlain": emoji.ReplaceAliases,
"ReactionToEmoji": ReactionToEmoji,
"RenderNote": RenderNote,
- "IsMultilineCommitMessage": IsMultilineCommitMessage,
+ "RenderMarkdownToHtml": func(input string) template.HTML {
+ output, err := markdown.RenderString(&markup.RenderContext{
+ URLPrefix: setting.AppSubURL,
+ }, input)
+ if err != nil {
+ log.Error("RenderString: %v", err)
+ }
+ return template.HTML(output)
+ },
+ "IsMultilineCommitMessage": IsMultilineCommitMessage,
"ThemeColorMetaTag": func() string {
return setting.UI.ThemeColorMetaTag
},
diff --git a/modules/util/filebuffer/file_backed_buffer.go b/modules/util/filebuffer/file_backed_buffer.go
new file mode 100644
index 0000000000..128030b4c5
--- /dev/null
+++ b/modules/util/filebuffer/file_backed_buffer.go
@@ -0,0 +1,147 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package filebuffer
+
+import (
+ "bytes"
+ "errors"
+ "io"
+ "os"
+)
+
+const maxInt = int(^uint(0) >> 1) // taken from bytes.Buffer
+
+var (
+ // ErrInvalidMemorySize occurs if the memory size is not in a valid range
+ ErrInvalidMemorySize = errors.New("Memory size must be greater 0 and lower math.MaxInt32")
+ // ErrWriteAfterRead occurs if Write is called after a read operation
+ ErrWriteAfterRead = errors.New("Write is unsupported after a read operation")
+)
+
+type readAtSeeker interface {
+ io.ReadSeeker
+ io.ReaderAt
+}
+
+// FileBackedBuffer uses a memory buffer with a fixed size.
+// If more data is written a temporary file is used instead.
+// It implements io.ReadWriteCloser, io.ReadSeekCloser and io.ReaderAt
+type FileBackedBuffer struct {
+ maxMemorySize int64
+ size int64
+ buffer bytes.Buffer
+ file *os.File
+ reader readAtSeeker
+}
+
+// New creates a file backed buffer with a specific maximum memory size
+func New(maxMemorySize int) (*FileBackedBuffer, error) {
+ if maxMemorySize < 0 || maxMemorySize > maxInt {
+ return nil, ErrInvalidMemorySize
+ }
+
+ return &FileBackedBuffer{
+ maxMemorySize: int64(maxMemorySize),
+ }, nil
+}
+
+// CreateFromReader creates a file backed buffer and copies the provided reader data into it.
+func CreateFromReader(r io.Reader, maxMemorySize int) (*FileBackedBuffer, error) {
+ b, err := New(maxMemorySize)
+ if err != nil {
+ return nil, err
+ }
+
+ _, err = io.Copy(b, r)
+ if err != nil {
+ return nil, err
+ }
+
+ return b, nil
+}
+
+// Write implements io.Writer
+func (b *FileBackedBuffer) Write(p []byte) (int, error) {
+ if b.reader != nil {
+ return 0, ErrWriteAfterRead
+ }
+
+ var n int
+ var err error
+
+ if b.file != nil {
+ n, err = b.file.Write(p)
+ } else {
+ if b.size+int64(len(p)) > b.maxMemorySize {
+ b.file, err = os.CreateTemp("", "gitea-buffer-")
+ if err != nil {
+ return 0, err
+ }
+
+ _, err = io.Copy(b.file, &b.buffer)
+ if err != nil {
+ return 0, err
+ }
+
+ return b.Write(p)
+ }
+
+ n, err = b.buffer.Write(p)
+ }
+
+ if err != nil {
+ return n, err
+ }
+ b.size += int64(n)
+ return n, nil
+}
+
+// Size returns the byte size of the buffered data
+func (b *FileBackedBuffer) Size() int64 {
+ return b.size
+}
+
+func (b *FileBackedBuffer) switchToReader() {
+ if b.reader != nil {
+ return
+ }
+
+ if b.file != nil {
+ b.reader = b.file
+ } else {
+ b.reader = bytes.NewReader(b.buffer.Bytes())
+ }
+}
+
+// Read implements io.Reader
+func (b *FileBackedBuffer) Read(p []byte) (int, error) {
+ b.switchToReader()
+
+ return b.reader.Read(p)
+}
+
+// ReadAt implements io.ReaderAt
+func (b *FileBackedBuffer) ReadAt(p []byte, off int64) (int, error) {
+ b.switchToReader()
+
+ return b.reader.ReadAt(p, off)
+}
+
+// Seek implements io.Seeker
+func (b *FileBackedBuffer) Seek(offset int64, whence int) (int64, error) {
+ b.switchToReader()
+
+ return b.reader.Seek(offset, whence)
+}
+
+// Close implements io.Closer
+func (b *FileBackedBuffer) Close() error {
+ if b.file != nil {
+ err := b.file.Close()
+ os.Remove(b.file.Name())
+ return err
+ }
+ return nil
+}