diff options
Diffstat (limited to 'modules')
48 files changed, 4369 insertions, 24 deletions
diff --git a/modules/context/context.go b/modules/context/context.go index eb0edef394..4905e1cb80 100644 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -70,6 +70,7 @@ type Context struct { ContextUser *user_model.User Repo *Repository Org *Organization + Package *Package } // TrHTMLEscapeArgs runs Tr but pre-escapes all arguments with html.EscapeString. @@ -331,6 +332,18 @@ func (ctx *Context) RespHeader() http.Header { return ctx.Resp.Header() } +// SetServeHeaders sets necessary content serve headers +func (ctx *Context) SetServeHeaders(filename string) { + ctx.Resp.Header().Set("Content-Description", "File Transfer") + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+filename) + ctx.Resp.Header().Set("Content-Transfer-Encoding", "binary") + ctx.Resp.Header().Set("Expires", "0") + ctx.Resp.Header().Set("Cache-Control", "must-revalidate") + ctx.Resp.Header().Set("Pragma", "public") + ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition") +} + // ServeContent serves content to http request func (ctx *Context) ServeContent(name string, r io.ReadSeeker, params ...interface{}) { modTime := time.Now() @@ -340,14 +353,7 @@ func (ctx *Context) ServeContent(name string, r io.ReadSeeker, params ...interfa modTime = v } } - ctx.Resp.Header().Set("Content-Description", "File Transfer") - ctx.Resp.Header().Set("Content-Type", "application/octet-stream") - ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+name) - ctx.Resp.Header().Set("Content-Transfer-Encoding", "binary") - ctx.Resp.Header().Set("Expires", "0") - ctx.Resp.Header().Set("Cache-Control", "must-revalidate") - ctx.Resp.Header().Set("Pragma", "public") - ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition") + ctx.SetServeHeaders(name) http.ServeContent(ctx.Resp, ctx.Req, name, modTime, r) } @@ -359,31 +365,41 @@ func (ctx *Context) ServeFile(file string, names ...string) { } else { name = path.Base(file) } - ctx.Resp.Header().Set("Content-Description", "File Transfer") - ctx.Resp.Header().Set("Content-Type", "application/octet-stream") - ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+name) - ctx.Resp.Header().Set("Content-Transfer-Encoding", "binary") - ctx.Resp.Header().Set("Expires", "0") - ctx.Resp.Header().Set("Cache-Control", "must-revalidate") - ctx.Resp.Header().Set("Pragma", "public") + ctx.SetServeHeaders(name) http.ServeFile(ctx.Resp, ctx.Req, file) } // ServeStream serves file via io stream func (ctx *Context) ServeStream(rd io.Reader, name string) { - ctx.Resp.Header().Set("Content-Description", "File Transfer") - ctx.Resp.Header().Set("Content-Type", "application/octet-stream") - ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+name) - ctx.Resp.Header().Set("Content-Transfer-Encoding", "binary") - ctx.Resp.Header().Set("Expires", "0") - ctx.Resp.Header().Set("Cache-Control", "must-revalidate") - ctx.Resp.Header().Set("Pragma", "public") + ctx.SetServeHeaders(name) _, err := io.Copy(ctx.Resp, rd) if err != nil { ctx.ServerError("Download file failed", err) } } +// UploadStream returns the request body or the first form file +// Only form files need to get closed. +func (ctx *Context) UploadStream() (rd io.ReadCloser, needToClose bool, err error) { + contentType := strings.ToLower(ctx.Req.Header.Get("Content-Type")) + if strings.HasPrefix(contentType, "application/x-www-form-urlencoded") || strings.HasPrefix(contentType, "multipart/form-data") { + if err := ctx.Req.ParseMultipartForm(32 << 20); err != nil { + return nil, false, err + } + if ctx.Req.MultipartForm.File == nil { + return nil, false, http.ErrMissingFile + } + for _, files := range ctx.Req.MultipartForm.File { + if len(files) > 0 { + r, err := files[0].Open() + return r, true, err + } + } + return nil, false, http.ErrMissingFile + } + return ctx.Req.Body, false, nil +} + // Error returned an error to web browser func (ctx *Context) Error(status int, contents ...string) { v := http.StatusText(status) diff --git a/modules/context/package.go b/modules/context/package.go new file mode 100644 index 0000000000..47af88c97b --- /dev/null +++ b/modules/context/package.go @@ -0,0 +1,109 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package context + +import ( + "fmt" + "net/http" + + "code.gitea.io/gitea/models/organization" + packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/models/perm" + user_model "code.gitea.io/gitea/models/user" +) + +// Package contains owner, access mode and optional the package descriptor +type Package struct { + Owner *user_model.User + AccessMode perm.AccessMode + Descriptor *packages_model.PackageDescriptor +} + +// PackageAssignment returns a middleware to handle Context.Package assignment +func PackageAssignment() func(ctx *Context) { + return func(ctx *Context) { + packageAssignment(ctx, func(status int, title string, obj interface{}) { + err, ok := obj.(error) + if !ok { + err = fmt.Errorf("%s", obj) + } + if status == http.StatusNotFound { + ctx.NotFound(title, err) + } else { + ctx.ServerError(title, err) + } + }) + } +} + +// PackageAssignmentAPI returns a middleware to handle Context.Package assignment +func PackageAssignmentAPI() func(ctx *APIContext) { + return func(ctx *APIContext) { + packageAssignment(ctx.Context, ctx.Error) + } +} + +func packageAssignment(ctx *Context, errCb func(int, string, interface{})) { + ctx.Package = &Package{ + Owner: ctx.ContextUser, + } + + if ctx.Doer != nil && ctx.Doer.ID == ctx.ContextUser.ID { + ctx.Package.AccessMode = perm.AccessModeOwner + } else { + if ctx.Package.Owner.IsOrganization() { + if organization.HasOrgOrUserVisible(ctx, ctx.Package.Owner, ctx.Doer) { + ctx.Package.AccessMode = perm.AccessModeRead + if ctx.Doer != nil { + var err error + ctx.Package.AccessMode, err = organization.OrgFromUser(ctx.Package.Owner).GetOrgUserMaxAuthorizeLevel(ctx.Doer.ID) + if err != nil { + errCb(http.StatusInternalServerError, "GetOrgUserMaxAuthorizeLevel", err) + return + } + } + } + } else { + ctx.Package.AccessMode = perm.AccessModeRead + } + } + + packageType := ctx.Params("type") + name := ctx.Params("name") + version := ctx.Params("version") + if packageType != "" && name != "" && version != "" { + pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.Type(packageType), name, version) + if err != nil { + if err == packages_model.ErrPackageNotExist { + errCb(http.StatusNotFound, "GetVersionByNameAndVersion", err) + } else { + errCb(http.StatusInternalServerError, "GetVersionByNameAndVersion", err) + } + return + } + + ctx.Package.Descriptor, err = packages_model.GetPackageDescriptor(ctx, pv) + if err != nil { + errCb(http.StatusInternalServerError, "GetPackageDescriptor", err) + return + } + } +} + +// PackageContexter initializes a package context for a request. +func PackageContexter() func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + ctx := Context{ + Resp: NewResponse(resp), + Data: map[string]interface{}{}, + } + + ctx.Req = WithContext(req, &ctx) + + next.ServeHTTP(ctx.Resp, ctx.Req) + }) + } +} diff --git a/modules/convert/package.go b/modules/convert/package.go new file mode 100644 index 0000000000..681219ca1a --- /dev/null +++ b/modules/convert/package.go @@ -0,0 +1,43 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package convert + +import ( + "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/models/perm" + api "code.gitea.io/gitea/modules/structs" +) + +// ToPackage convert a packages.PackageDescriptor to api.Package +func ToPackage(pd *packages.PackageDescriptor) *api.Package { + var repo *api.Repository + if pd.Repository != nil { + repo = ToRepo(pd.Repository, perm.AccessModeNone) + } + + return &api.Package{ + ID: pd.Version.ID, + Owner: ToUser(pd.Owner, nil), + Repository: repo, + Creator: ToUser(pd.Creator, nil), + Type: string(pd.Package.Type), + Name: pd.Package.Name, + Version: pd.Version.Version, + CreatedAt: pd.Version.CreatedUnix.AsTime(), + } +} + +// ToPackageFile converts packages.PackageFileDescriptor to api.PackageFile +func ToPackageFile(pfd *packages.PackageFileDescriptor) *api.PackageFile { + return &api.PackageFile{ + ID: pfd.File.ID, + Size: pfd.Blob.Size, + Name: pfd.File.Name, + HashMD5: pfd.Blob.HashMD5, + HashSHA1: pfd.Blob.HashSHA1, + HashSHA256: pfd.Blob.HashSHA256, + HashSHA512: pfd.Blob.HashSHA512, + } +} diff --git a/modules/notification/base/notifier.go b/modules/notification/base/notifier.go index 8174741169..2b8be18ad3 100644 --- a/modules/notification/base/notifier.go +++ b/modules/notification/base/notifier.go @@ -6,6 +6,7 @@ package base import ( "code.gitea.io/gitea/models" + packages_model "code.gitea.io/gitea/models/packages" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/repository" @@ -54,4 +55,6 @@ type Notifier interface { NotifySyncCreateRef(doer *user_model.User, repo *repo_model.Repository, refType, refFullName, refID string) NotifySyncDeleteRef(doer *user_model.User, repo *repo_model.Repository, refType, refFullName string) NotifyRepoPendingTransfer(doer, newOwner *user_model.User, repo *repo_model.Repository) + NotifyPackageCreate(doer *user_model.User, pd *packages_model.PackageDescriptor) + NotifyPackageDelete(doer *user_model.User, pd *packages_model.PackageDescriptor) } diff --git a/modules/notification/base/null.go b/modules/notification/base/null.go index 2bfcaafda9..29b5f0c97e 100644 --- a/modules/notification/base/null.go +++ b/modules/notification/base/null.go @@ -6,6 +6,7 @@ package base import ( "code.gitea.io/gitea/models" + packages_model "code.gitea.io/gitea/models/packages" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/repository" @@ -173,3 +174,11 @@ func (*NullNotifier) NotifySyncDeleteRef(doer *user_model.User, repo *repo_model // NotifyRepoPendingTransfer places a place holder function func (*NullNotifier) NotifyRepoPendingTransfer(doer, newOwner *user_model.User, repo *repo_model.Repository) { } + +// NotifyPackageCreate places a place holder function +func (*NullNotifier) NotifyPackageCreate(doer *user_model.User, pd *packages_model.PackageDescriptor) { +} + +// NotifyPackageDelete places a place holder function +func (*NullNotifier) NotifyPackageDelete(doer *user_model.User, pd *packages_model.PackageDescriptor) { +} diff --git a/modules/notification/notification.go b/modules/notification/notification.go index a31e3810e2..90ff87941f 100644 --- a/modules/notification/notification.go +++ b/modules/notification/notification.go @@ -6,6 +6,7 @@ package notification import ( "code.gitea.io/gitea/models" + packages_model "code.gitea.io/gitea/models/packages" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/notification/action" @@ -306,3 +307,17 @@ func NotifyRepoPendingTransfer(doer, newOwner *user_model.User, repo *repo_model notifier.NotifyRepoPendingTransfer(doer, newOwner, repo) } } + +// NotifyPackageCreate notifies creation of a package to notifiers +func NotifyPackageCreate(doer *user_model.User, pd *packages_model.PackageDescriptor) { + for _, notifier := range notifiers { + notifier.NotifyPackageCreate(doer, pd) + } +} + +// NotifyPackageDelete notifies deletion of a package to notifiers +func NotifyPackageDelete(doer *user_model.User, pd *packages_model.PackageDescriptor) { + for _, notifier := range notifiers { + notifier.NotifyPackageDelete(doer, pd) + } +} diff --git a/modules/notification/webhook/webhook.go b/modules/notification/webhook/webhook.go index d4d5eea6cb..94d4d180be 100644 --- a/modules/notification/webhook/webhook.go +++ b/modules/notification/webhook/webhook.go @@ -8,6 +8,7 @@ import ( "fmt" "code.gitea.io/gitea/models" + packages_model "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/models/perm" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" @@ -855,3 +856,33 @@ func (m *webhookNotifier) NotifySyncCreateRef(pusher *user_model.User, repo *rep func (m *webhookNotifier) NotifySyncDeleteRef(pusher *user_model.User, repo *repo_model.Repository, refType, refFullName string) { m.NotifyDeleteRef(pusher, repo, refType, refFullName) } + +func (m *webhookNotifier) NotifyPackageCreate(doer *user_model.User, pd *packages_model.PackageDescriptor) { + notifyPackage(doer, pd, api.HookPackageCreated) +} + +func (m *webhookNotifier) NotifyPackageDelete(doer *user_model.User, pd *packages_model.PackageDescriptor) { + notifyPackage(doer, pd, api.HookPackageDeleted) +} + +func notifyPackage(sender *user_model.User, pd *packages_model.PackageDescriptor, action api.HookPackageAction) { + if pd.Repository == nil { + // TODO https://github.com/go-gitea/gitea/pull/17940 + return + } + + org := pd.Owner + if !org.IsOrganization() { + org = nil + } + + if err := webhook_services.PrepareWebhooks(pd.Repository, webhook.HookEventPackage, &api.PackagePayload{ + Action: action, + Repository: convert.ToRepo(pd.Repository, perm.AccessModeNone), + Package: convert.ToPackage(pd), + Organization: convert.ToUser(org, nil), + Sender: convert.ToUser(sender, nil), + }); err != nil { + log.Error("PrepareWebhooks: %v", err) + } +} diff --git a/modules/packages/composer/metadata.go b/modules/packages/composer/metadata.go new file mode 100644 index 0000000000..797576b1e7 --- /dev/null +++ b/modules/packages/composer/metadata.go @@ -0,0 +1,147 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package composer + +import ( + "archive/zip" + "errors" + "io" + "regexp" + "strings" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/validation" + + "github.com/hashicorp/go-version" +) + +// TypeProperty is the name of the property for Composer package types +const TypeProperty = "composer.type" + +var ( + // ErrMissingComposerFile indicates a missing composer.json file + ErrMissingComposerFile = errors.New("composer.json file is missing") + // ErrInvalidName indicates an invalid package name + ErrInvalidName = errors.New("package name is invalid") + // ErrInvalidVersion indicates an invalid package version + ErrInvalidVersion = errors.New("package version is invalid") +) + +// Package represents a Composer package +type Package struct { + Name string + Version string + Type string + Metadata *Metadata +} + +// Metadata represents the metadata of a Composer package +type Metadata struct { + Description string `json:"description,omitempty"` + Keywords []string `json:"keywords,omitempty"` + Homepage string `json:"homepage,omitempty"` + License Licenses `json:"license,omitempty"` + Authors []Author `json:"authors,omitempty"` + Autoload map[string]interface{} `json:"autoload,omitempty"` + AutoloadDev map[string]interface{} `json:"autoload-dev,omitempty"` + Extra map[string]interface{} `json:"extra,omitempty"` + Require map[string]string `json:"require,omitempty"` + RequireDev map[string]string `json:"require-dev,omitempty"` + Suggest map[string]string `json:"suggest,omitempty"` + Provide map[string]string `json:"provide,omitempty"` +} + +// Licenses represents the licenses of a Composer package +type Licenses []string + +// UnmarshalJSON reads from a string or array +func (l *Licenses) UnmarshalJSON(data []byte) error { + switch data[0] { + case '"': + var value string + if err := json.Unmarshal(data, &value); err != nil { + return err + } + *l = Licenses{value} + case '[': + values := make([]string, 0, 5) + if err := json.Unmarshal(data, &values); err != nil { + return err + } + *l = Licenses(values) + } + return nil +} + +// Author represents an author +type Author struct { + Name string `json:"name,omitempty"` + Email string `json:"email,omitempty"` + Homepage string `json:"homepage,omitempty"` +} + +var nameMatch = regexp.MustCompile(`\A[a-z0-9]([_\.-]?[a-z0-9]+)*/[a-z0-9](([_\.]?|-{0,2})[a-z0-9]+)*\z`) + +// ParsePackage parses the metadata of a Composer package file +func ParsePackage(r io.ReaderAt, size int64) (*Package, error) { + archive, err := zip.NewReader(r, size) + if err != nil { + return nil, err + } + + for _, file := range archive.File { + if strings.Count(file.Name, "/") > 1 { + continue + } + if strings.HasSuffix(strings.ToLower(file.Name), "composer.json") { + f, err := archive.Open(file.Name) + if err != nil { + return nil, err + } + defer f.Close() + + return ParseComposerFile(f) + } + } + return nil, ErrMissingComposerFile +} + +// ParseComposerFile parses a composer.json file to retrieve the metadata of a Composer package +func ParseComposerFile(r io.Reader) (*Package, error) { + var cj struct { + Name string `json:"name"` + Version string `json:"version"` + Type string `json:"type"` + Metadata + } + if err := json.NewDecoder(r).Decode(&cj); err != nil { + return nil, err + } + + if !nameMatch.MatchString(cj.Name) { + return nil, ErrInvalidName + } + + if cj.Version != "" { + if _, err := version.NewSemver(cj.Version); err != nil { + return nil, ErrInvalidVersion + } + } + + if !validation.IsValidURL(cj.Homepage) { + cj.Homepage = "" + } + + if cj.Type == "" { + cj.Type = "library" + } + + return &Package{ + Name: cj.Name, + Version: cj.Version, + Type: cj.Type, + Metadata: &cj.Metadata, + }, nil +} diff --git a/modules/packages/composer/metadata_test.go b/modules/packages/composer/metadata_test.go new file mode 100644 index 0000000000..feadc18b6a --- /dev/null +++ b/modules/packages/composer/metadata_test.go @@ -0,0 +1,130 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package composer + +import ( + "archive/zip" + "bytes" + "strings" + "testing" + + "code.gitea.io/gitea/modules/json" + + "github.com/stretchr/testify/assert" +) + +const ( + name = "gitea/composer-package" + description = "Package Description" + packageType = "composer-plugin" + author = "Gitea Authors" + email = "no.reply@gitea.io" + homepage = "https://gitea.io" + license = "MIT" +) + +const composerContent = `{ + "name": "` + name + `", + "description": "` + description + `", + "type": "` + packageType + `", + "license": "` + license + `", + "authors": [ + { + "name": "` + author + `", + "email": "` + email + `" + } + ], + "homepage": "` + homepage + `", + "autoload": { + "psr-4": {"Gitea\\ComposerPackage\\": "src/"} + }, + "require": { + "php": ">=7.2 || ^8.0" + } +}` + +func TestLicenseUnmarshal(t *testing.T) { + var l Licenses + assert.NoError(t, json.NewDecoder(strings.NewReader(`["MIT"]`)).Decode(&l)) + assert.Len(t, l, 1) + assert.Equal(t, "MIT", l[0]) + assert.NoError(t, json.NewDecoder(strings.NewReader(`"MIT"`)).Decode(&l)) + assert.Len(t, l, 1) + assert.Equal(t, "MIT", l[0]) +} + +func TestParsePackage(t *testing.T) { + createArchive := func(name, content string) []byte { + var buf bytes.Buffer + archive := zip.NewWriter(&buf) + w, _ := archive.Create(name) + w.Write([]byte(content)) + archive.Close() + return buf.Bytes() + } + + t.Run("MissingComposerFile", func(t *testing.T) { + data := createArchive("dummy.txt", "") + + cp, err := ParsePackage(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, cp) + assert.ErrorIs(t, err, ErrMissingComposerFile) + }) + + t.Run("MissingComposerFileInRoot", func(t *testing.T) { + data := createArchive("sub/sub/composer.json", "") + + cp, err := ParsePackage(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, cp) + assert.ErrorIs(t, err, ErrMissingComposerFile) + }) + + t.Run("InvalidComposerFile", func(t *testing.T) { + data := createArchive("composer.json", "") + + cp, err := ParsePackage(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, cp) + assert.Error(t, err) + }) + + t.Run("Valid", func(t *testing.T) { + data := createArchive("composer.json", composerContent) + + cp, err := ParsePackage(bytes.NewReader(data), int64(len(data))) + assert.NoError(t, err) + assert.NotNil(t, cp) + }) +} + +func TestParseComposerFile(t *testing.T) { + t.Run("InvalidPackageName", func(t *testing.T) { + cp, err := ParseComposerFile(strings.NewReader(`{}`)) + assert.Nil(t, cp) + assert.ErrorIs(t, err, ErrInvalidName) + }) + + t.Run("InvalidPackageVersion", func(t *testing.T) { + cp, err := ParseComposerFile(strings.NewReader(`{"name": "gitea/composer-package", "version": "1.a.3"}`)) + assert.Nil(t, cp) + assert.ErrorIs(t, err, ErrInvalidVersion) + }) + + t.Run("Valid", func(t *testing.T) { + cp, err := ParseComposerFile(strings.NewReader(composerContent)) + assert.NoError(t, err) + assert.NotNil(t, cp) + + assert.Equal(t, name, cp.Name) + assert.Empty(t, cp.Version) + assert.Equal(t, description, cp.Metadata.Description) + assert.Len(t, cp.Metadata.Authors, 1) + assert.Equal(t, author, cp.Metadata.Authors[0].Name) + assert.Equal(t, email, cp.Metadata.Authors[0].Email) + assert.Equal(t, homepage, cp.Metadata.Homepage) + assert.Equal(t, packageType, cp.Type) + assert.Len(t, cp.Metadata.License, 1) + assert.Equal(t, license, cp.Metadata.License[0]) + }) +} diff --git a/modules/packages/conan/conanfile_parser.go b/modules/packages/conan/conanfile_parser.go new file mode 100644 index 0000000000..960e813533 --- /dev/null +++ b/modules/packages/conan/conanfile_parser.go @@ -0,0 +1,68 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package conan + +import ( + "io" + "regexp" + "strings" +) + +var ( + patternAuthor = compilePattern("author") + patternHomepage = compilePattern("homepage") + patternURL = compilePattern("url") + patternLicense = compilePattern("license") + patternDescription = compilePattern("description") + patternTopics = regexp.MustCompile(`(?im)^\s*topics\s*=\s*\((.+)\)`) + patternTopicList = regexp.MustCompile(`\s*['"](.+?)['"]\s*,?`) +) + +func compilePattern(name string) *regexp.Regexp { + return regexp.MustCompile(`(?im)^\s*` + name + `\s*=\s*['"\(](.+)['"\)]`) +} + +func ParseConanfile(r io.Reader) (*Metadata, error) { + buf, err := io.ReadAll(io.LimitReader(r, 1<<20)) + if err != nil { + return nil, err + } + + metadata := &Metadata{} + + m := patternAuthor.FindSubmatch(buf) + if len(m) > 1 && len(m[1]) > 0 { + metadata.Author = string(m[1]) + } + m = patternHomepage.FindSubmatch(buf) + if len(m) > 1 && len(m[1]) > 0 { + metadata.ProjectURL = string(m[1]) + } + m = patternURL.FindSubmatch(buf) + if len(m) > 1 && len(m[1]) > 0 { + metadata.RepositoryURL = string(m[1]) + } + m = patternLicense.FindSubmatch(buf) + if len(m) > 1 && len(m[1]) > 0 { + metadata.License = strings.ReplaceAll(strings.ReplaceAll(string(m[1]), "'", ""), "\"", "") + } + m = patternDescription.FindSubmatch(buf) + if len(m) > 1 && len(m[1]) > 0 { + metadata.Description = string(m[1]) + } + m = patternTopics.FindSubmatch(buf) + if len(m) > 1 && len(m[1]) > 0 { + m2 := patternTopicList.FindAllSubmatch(m[1], -1) + if len(m2) > 0 { + metadata.Keywords = make([]string, 0, len(m2)) + for _, g := range m2 { + if len(g) > 1 { + metadata.Keywords = append(metadata.Keywords, string(g[1])) + } + } + } + } + return metadata, nil +} diff --git a/modules/packages/conan/conanfile_parser_test.go b/modules/packages/conan/conanfile_parser_test.go new file mode 100644 index 0000000000..0ac9c87b14 --- /dev/null +++ b/modules/packages/conan/conanfile_parser_test.go @@ -0,0 +1,51 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package conan + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + name = "ConanPackage" + version = "1.2" + license = "MIT" + author = "Gitea <info@gitea.io>" + homepage = "https://gitea.io/" + url = "https://gitea.com/" + description = "Description of ConanPackage" + topic1 = "gitea" + topic2 = "conan" + contentConanfile = `from conans import ConanFile, CMake, tools + +class ConanPackageConan(ConanFile): + name = "` + name + `" + version = "` + version + `" + license = "` + license + `" + author = "` + author + `" + homepage = "` + homepage + `" + url = "` + url + `" + description = "` + description + `" + topics = ("` + topic1 + `", "` + topic2 + `") + settings = "os", "compiler", "build_type", "arch" + options = {"shared": [True, False], "fPIC": [True, False]} + default_options = {"shared": False, "fPIC": True} + generators = "cmake" +` +) + +func TestParseConanfile(t *testing.T) { + metadata, err := ParseConanfile(strings.NewReader(contentConanfile)) + assert.Nil(t, err) + assert.Equal(t, license, metadata.License) + assert.Equal(t, author, metadata.Author) + assert.Equal(t, homepage, metadata.ProjectURL) + assert.Equal(t, url, metadata.RepositoryURL) + assert.Equal(t, description, metadata.Description) + assert.Equal(t, []string{topic1, topic2}, metadata.Keywords) +} diff --git a/modules/packages/conan/conaninfo_parser.go b/modules/packages/conan/conaninfo_parser.go new file mode 100644 index 0000000000..bb228e0207 --- /dev/null +++ b/modules/packages/conan/conaninfo_parser.go @@ -0,0 +1,123 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package conan + +import ( + "bufio" + "errors" + "io" + "strings" +) + +// Conaninfo represents infos of a Conan package +type Conaninfo struct { + Settings map[string]string `json:"settings"` + FullSettings map[string]string `json:"full_settings"` + Requires []string `json:"requires"` + FullRequires []string `json:"full_requires"` + Options map[string]string `json:"options"` + FullOptions map[string]string `json:"full_options"` + RecipeHash string `json:"recipe_hash"` + Environment map[string][]string `json:"environment"` +} + +func ParseConaninfo(r io.Reader) (*Conaninfo, error) { + sections, err := readSections(io.LimitReader(r, 1<<20)) + if err != nil { + return nil, err + } + + info := &Conaninfo{} + for section, lines := range sections { + if len(lines) == 0 { + continue + } + switch section { + case "settings": + info.Settings = toMap(lines) + case "full_settings": + info.FullSettings = toMap(lines) + case "options": + info.Options = toMap(lines) + case "full_options": + info.FullOptions = toMap(lines) + case "requires": + info.Requires = lines + case "full_requires": + info.FullRequires = lines + case "recipe_hash": + info.RecipeHash = lines[0] + case "env": + info.Environment = toMapArray(lines) + } + } + return info, nil +} + +func readSections(r io.Reader) (map[string][]string, error) { + sections := make(map[string][]string) + + section := "" + lines := make([]string, 0, 5) + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") { + if section != "" { + sections[section] = lines + } + section = line[1 : len(line)-1] + lines = make([]string, 0, 5) + continue + } + if section != "" { + if line != "" { + lines = append(lines, line) + } + continue + } + if line != "" { + return nil, errors.New("Invalid conaninfo.txt") + } + } + if err := scanner.Err(); err != nil { + return nil, err + } + if section != "" { + sections[section] = lines + } + return sections, nil +} + +func toMap(lines []string) map[string]string { + result := make(map[string]string) + for _, line := range lines { + parts := strings.SplitN(line, "=", 2) + if len(parts) != 2 || len(parts[0]) == 0 || len(parts[1]) == 0 { + continue + } + result[parts[0]] = parts[1] + } + return result +} + +func toMapArray(lines []string) map[string][]string { + result := make(map[string][]string) + for _, line := range lines { + parts := strings.SplitN(line, "=", 2) + if len(parts) != 2 || len(parts[0]) == 0 || len(parts[1]) == 0 { + continue + } + var items []string + if strings.HasPrefix(parts[1], "[") && strings.HasSuffix(parts[1], "]") { + items = strings.Split(parts[1], ",") + } else { + items = []string{parts[1]} + } + result[parts[0]] = items + } + return result +} diff --git a/modules/packages/conan/conaninfo_parser_test.go b/modules/packages/conan/conaninfo_parser_test.go new file mode 100644 index 0000000000..3e28191b06 --- /dev/null +++ b/modules/packages/conan/conaninfo_parser_test.go @@ -0,0 +1,85 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package conan + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + settingsKey = "arch" + settingsValue = "x84_64" + optionsKey = "shared" + optionsValue = "False" + requires = "fmt/7.1.3" + hash = "74714915a51073acb548ca1ce29afbac" + envKey = "CC" + envValue = "gcc-10" + + contentConaninfo = `[settings] + ` + settingsKey + `=` + settingsValue + ` + +[requires] + ` + requires + ` + +[options] + ` + optionsKey + `=` + optionsValue + ` + +[full_settings] + ` + settingsKey + `=` + settingsValue + ` + +[full_requires] + ` + requires + ` + +[full_options] + ` + optionsKey + `=` + optionsValue + ` + +[recipe_hash] + ` + hash + ` + +[env] +` + envKey + `=` + envValue + ` + +` +) + +func TestParseConaninfo(t *testing.T) { + info, err := ParseConaninfo(strings.NewReader(contentConaninfo)) + assert.NotNil(t, info) + assert.Nil(t, err) + assert.Equal( + t, + map[string]string{ + settingsKey: settingsValue, + }, + info.Settings, + ) + assert.Equal(t, info.Settings, info.FullSettings) + assert.Equal( + t, + map[string]string{ + optionsKey: optionsValue, + }, + info.Options, + ) + assert.Equal(t, info.Options, info.FullOptions) + assert.Equal( + t, + []string{requires}, + info.Requires, + ) + assert.Equal(t, info.Requires, info.FullRequires) + assert.Equal(t, hash, info.RecipeHash) + assert.Equal( + t, + map[string][]string{ + envKey: {envValue}, + }, + info.Environment, + ) +} diff --git a/modules/packages/conan/metadata.go b/modules/packages/conan/metadata.go new file mode 100644 index 0000000000..a7d6a9df0b --- /dev/null +++ b/modules/packages/conan/metadata.go @@ -0,0 +1,24 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package conan + +const ( + PropertyRecipeUser = "conan.recipe.user" + PropertyRecipeChannel = "conan.recipe.channel" + PropertyRecipeRevision = "conan.recipe.revision" + PropertyPackageReference = "conan.package.reference" + PropertyPackageRevision = "conan.package.revision" + PropertyPackageInfo = "conan.package.info" +) + +// Metadata represents the metadata of a Conan package +type Metadata struct { + Author string `json:"author,omitempty"` + License string `json:"license,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + Description string `json:"description,omitempty"` + Keywords []string `json:"keywords,omitempty"` +} diff --git a/modules/packages/conan/reference.go b/modules/packages/conan/reference.go new file mode 100644 index 0000000000..c43446e6e5 --- /dev/null +++ b/modules/packages/conan/reference.go @@ -0,0 +1,155 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package conan + +import ( + "errors" + "fmt" + "regexp" + + "code.gitea.io/gitea/modules/log" + + goversion "github.com/hashicorp/go-version" +) + +const ( + // taken from https://github.com/conan-io/conan/blob/develop/conans/model/ref.py + minChars = 2 + maxChars = 51 + + // DefaultRevision if no revision is specified + DefaultRevision = "0" +) + +var ( + namePattern = regexp.MustCompile(fmt.Sprintf(`^[a-zA-Z0-9_][a-zA-Z0-9_\+\.-]{%d,%d}$`, minChars-1, maxChars-1)) + revisionPattern = regexp.MustCompile(fmt.Sprintf(`^[a-zA-Z0-9]{1,%d}$`, maxChars)) + + ErrValidation = errors.New("Could not validate one or more reference fields") +) + +// RecipeReference represents a recipe <Name>/<Version>@<User>/<Channel>#<Revision> +type RecipeReference struct { + Name string + Version string + User string + Channel string + Revision string +} + +func NewRecipeReference(name, version, user, channel, revision string) (*RecipeReference, error) { + log.Trace("Conan Recipe: %s/%s(@%s/%s(#%s))", name, version, user, channel, revision) + + if user == "_" { + user = "" + } + if channel == "_" { + channel = "" + } + + if (user != "" && channel == "") || (user == "" && channel != "") { + return nil, ErrValidation + } + + if !namePattern.MatchString(name) { + return nil, ErrValidation + } + if _, err := goversion.NewSemver(version); err != nil { + return nil, ErrValidation + } + if user != "" && !namePattern.MatchString(user) { + return nil, ErrValidation + } + if channel != "" && !namePattern.MatchString(channel) { + return nil, ErrValidation + } + if revision != "" && !revisionPattern.MatchString(revision) { + return nil, ErrValidation + } + + return &RecipeReference{name, version, user, channel, revision}, nil +} + +func (r *RecipeReference) RevisionOrDefault() string { + if r.Revision == "" { + return DefaultRevision + } + return r.Revision +} + +func (r *RecipeReference) String() string { + rev := "" + if r.Revision != "" { + rev = "#" + r.Revision + } + if r.User == "" || r.Channel == "" { + return fmt.Sprintf("%s/%s%s", r.Name, r.Version, rev) + } + return fmt.Sprintf("%s/%s@%s/%s%s", r.Name, r.Version, r.User, r.Channel, rev) +} + +func (r *RecipeReference) LinkName() string { + user := r.User + if user == "" { + user = "_" + } + channel := r.Channel + if channel == "" { + channel = "_" + } + return fmt.Sprintf("%s/%s/%s/%s/%s", r.Name, r.Version, user, channel, r.RevisionOrDefault()) +} + +func (r *RecipeReference) WithRevision(revision string) *RecipeReference { + return &RecipeReference{r.Name, r.Version, r.User, r.Channel, revision} +} + +// AsKey builds the additional key for the package file +func (r *RecipeReference) AsKey() string { + return fmt.Sprintf("%s|%s|%s", r.User, r.Channel, r.RevisionOrDefault()) +} + +// PackageReference represents a package of a recipe <Name>/<Version>@<User>/<Channel>#<Revision> <Reference>#<Revision> +type PackageReference struct { + Recipe *RecipeReference + Reference string + Revision string +} + +func NewPackageReference(recipe *RecipeReference, reference, revision string) (*PackageReference, error) { + log.Trace("Conan Package: %v %s(#%s)", recipe, reference, revision) + + if recipe == nil { + return nil, ErrValidation + } + if reference == "" || !revisionPattern.MatchString(reference) { + return nil, ErrValidation + } + if revision != "" && !revisionPattern.MatchString(revision) { + return nil, ErrValidation + } + + return &PackageReference{recipe, reference, revision}, nil +} + +func (r *PackageReference) RevisionOrDefault() string { + if r.Revision == "" { + return DefaultRevision + } + return r.Revision +} + +func (r *PackageReference) LinkName() string { + return fmt.Sprintf("%s/%s", r.Reference, r.RevisionOrDefault()) +} + +func (r *PackageReference) WithRevision(revision string) *PackageReference { + return &PackageReference{r.Recipe, r.Reference, revision} +} + +// AsKey builds the additional key for the package file +func (r *PackageReference) AsKey() string { + return fmt.Sprintf("%s|%s|%s|%s|%s", r.Recipe.User, r.Recipe.Channel, r.Recipe.RevisionOrDefault(), r.Reference, r.RevisionOrDefault()) +} diff --git a/modules/packages/conan/reference_test.go b/modules/packages/conan/reference_test.go new file mode 100644 index 0000000000..29ba3a543b --- /dev/null +++ b/modules/packages/conan/reference_test.go @@ -0,0 +1,147 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package conan + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewRecipeReference(t *testing.T) { + cases := []struct { + Name string + Version string + User string + Channel string + Revision string + IsValid bool + }{ + {"", "", "", "", "", false}, + {"name", "", "", "", "", false}, + {"", "1.0", "", "", "", false}, + {"", "", "user", "", "", false}, + {"", "", "", "channel", "", false}, + {"", "", "", "", "0", false}, + {"name", "1.0", "", "", "", true}, + {"name", "1.0", "user", "", "", false}, + {"name", "1.0", "", "channel", "", false}, + {"name", "1.0", "user", "channel", "", true}, + {"name", "1.0", "_", "", "", true}, + {"name", "1.0", "", "_", "", true}, + {"name", "1.0", "_", "_", "", true}, + {"name", "1.0", "_", "_", "0", true}, + {"name", "1.0", "", "", "0", true}, + {"name", "1.0", "", "", "000000000000000000000000000000000000000000000000000000000000", false}, + } + + for i, c := range cases { + rref, err := NewRecipeReference(c.Name, c.Version, c.User, c.Channel, c.Revision) + if c.IsValid { + assert.NoError(t, err, "case %d, should be invalid", i) + assert.NotNil(t, rref, "case %d, should not be nil", i) + } else { + assert.Error(t, err, "case %d, should be valid", i) + } + } +} + +func TestRecipeReferenceRevisionOrDefault(t *testing.T) { + rref, err := NewRecipeReference("name", "1.0", "", "", "") + assert.NoError(t, err) + assert.Equal(t, DefaultRevision, rref.RevisionOrDefault()) + + rref, err = NewRecipeReference("name", "1.0", "", "", DefaultRevision) + assert.NoError(t, err) + assert.Equal(t, DefaultRevision, rref.RevisionOrDefault()) + + rref, err = NewRecipeReference("name", "1.0", "", "", "Az09") + assert.NoError(t, err) + assert.Equal(t, "Az09", rref.RevisionOrDefault()) +} + +func TestRecipeReferenceString(t *testing.T) { + rref, err := NewRecipeReference("name", "1.0", "", "", "") + assert.NoError(t, err) + assert.Equal(t, "name/1.0", rref.String()) + + rref, err = NewRecipeReference("name", "1.0", "user", "channel", "") + assert.NoError(t, err) + assert.Equal(t, "name/1.0@user/channel", rref.String()) + + rref, err = NewRecipeReference("name", "1.0", "user", "channel", "Az09") + assert.NoError(t, err) + assert.Equal(t, "name/1.0@user/channel#Az09", rref.String()) +} + +func TestRecipeReferenceLinkName(t *testing.T) { + rref, err := NewRecipeReference("name", "1.0", "", "", "") + assert.NoError(t, err) + assert.Equal(t, "name/1.0/_/_/0", rref.LinkName()) + + rref, err = NewRecipeReference("name", "1.0", "user", "channel", "") + assert.NoError(t, err) + assert.Equal(t, "name/1.0/user/channel/0", rref.LinkName()) + + rref, err = NewRecipeReference("name", "1.0", "user", "channel", "Az09") + assert.NoError(t, err) + assert.Equal(t, "name/1.0/user/channel/Az09", rref.LinkName()) +} + +func TestNewPackageReference(t *testing.T) { + rref, _ := NewRecipeReference("name", "1.0", "", "", "") + + cases := []struct { + Recipe *RecipeReference + Reference string + Revision string + IsValid bool + }{ + {nil, "", "", false}, + {rref, "", "", false}, + {nil, "aZ09", "", false}, + {rref, "aZ09", "", true}, + {rref, "", "Az09", false}, + {rref, "aZ09", "Az09", true}, + } + + for i, c := range cases { + pref, err := NewPackageReference(c.Recipe, c.Reference, c.Revision) + if c.IsValid { + assert.NoError(t, err, "case %d, should be invalid", i) + assert.NotNil(t, pref, "case %d, should not be nil", i) + } else { + assert.Error(t, err, "case %d, should be valid", i) + } + } +} + +func TestPackageReferenceRevisionOrDefault(t *testing.T) { + rref, _ := NewRecipeReference("name", "1.0", "", "", "") + + pref, err := NewPackageReference(rref, "ref", "") + assert.NoError(t, err) + assert.Equal(t, DefaultRevision, pref.RevisionOrDefault()) + + pref, err = NewPackageReference(rref, "ref", DefaultRevision) + assert.NoError(t, err) + assert.Equal(t, DefaultRevision, pref.RevisionOrDefault()) + + pref, err = NewPackageReference(rref, "ref", "Az09") + assert.NoError(t, err) + assert.Equal(t, "Az09", pref.RevisionOrDefault()) +} + +func TestPackageReferenceLinkName(t *testing.T) { + rref, _ := NewRecipeReference("name", "1.0", "", "", "") + + pref, err := NewPackageReference(rref, "ref", "") + assert.NoError(t, err) + assert.Equal(t, "ref/0", pref.LinkName()) + + pref, err = NewPackageReference(rref, "ref", "Az09") + assert.NoError(t, err) + assert.Equal(t, "ref/Az09", pref.LinkName()) +} diff --git a/modules/packages/container/helm/helm.go b/modules/packages/container/helm/helm.go new file mode 100644 index 0000000000..98d3824a85 --- /dev/null +++ b/modules/packages/container/helm/helm.go @@ -0,0 +1,56 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package helm + +// https://github.com/helm/helm/blob/main/pkg/chart/ + +const ConfigMediaType = "application/vnd.cncf.helm.config.v1+json" + +// Maintainer describes a Chart maintainer. +type Maintainer struct { + // Name is a user name or organization name + Name string `json:"name,omitempty"` + // Email is an optional email address to contact the named maintainer + Email string `json:"email,omitempty"` + // URL is an optional URL to an address for the named maintainer + URL string `json:"url,omitempty"` +} + +// Metadata for a Chart file. This models the structure of a Chart.yaml file. +type Metadata struct { + // The name of the chart. Required. + Name string `json:"name,omitempty"` + // The URL to a relevant project page, git repo, or contact person + Home string `json:"home,omitempty"` + // Source is the URL to the source code of this chart + Sources []string `json:"sources,omitempty"` + // A SemVer 2 conformant version string of the chart. Required. + Version string `json:"version,omitempty"` + // A one-sentence description of the chart + Description string `json:"description,omitempty"` + // A list of string keywords + Keywords []string `json:"keywords,omitempty"` + // A list of name and URL/email address combinations for the maintainer(s) + Maintainers []*Maintainer `json:"maintainers,omitempty"` + // The URL to an icon file. + Icon string `json:"icon,omitempty"` + // The API Version of this chart. Required. + APIVersion string `json:"apiVersion,omitempty"` + // The condition to check to enable chart + Condition string `json:"condition,omitempty"` + // The tags to check to enable chart + Tags string `json:"tags,omitempty"` + // The version of the application enclosed inside of this chart. + AppVersion string `json:"appVersion,omitempty"` + // Whether or not this chart is deprecated + Deprecated bool `json:"deprecated,omitempty"` + // Annotations are additional mappings uninterpreted by Helm, + // made available for inspection by other applications. + Annotations map[string]string `json:"annotations,omitempty"` + // KubeVersion is a SemVer constraint specifying the version of Kubernetes required. + KubeVersion string `json:"kubeVersion,omitempty"` + // Specifies the chart type: application or library + Type string `json:"type,omitempty"` +} diff --git a/modules/packages/container/metadata.go b/modules/packages/container/metadata.go new file mode 100644 index 0000000000..087d38e5bd --- /dev/null +++ b/modules/packages/container/metadata.go @@ -0,0 +1,157 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package container + +import ( + "fmt" + "io" + "strings" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/packages/container/helm" + "code.gitea.io/gitea/modules/packages/container/oci" + "code.gitea.io/gitea/modules/validation" +) + +const ( + PropertyDigest = "container.digest" + PropertyMediaType = "container.mediatype" + PropertyManifestTagged = "container.manifest.tagged" + PropertyManifestReference = "container.manifest.reference" + + DefaultPlatform = "linux/amd64" + + labelLicenses = "org.opencontainers.image.licenses" + labelURL = "org.opencontainers.image.url" + labelSource = "org.opencontainers.image.source" + labelDocumentation = "org.opencontainers.image.documentation" + labelDescription = "org.opencontainers.image.description" + labelAuthors = "org.opencontainers.image.authors" +) + +type ImageType string + +const ( + TypeOCI ImageType = "oci" + TypeHelm ImageType = "helm" +) + +// Name gets the name of the image type +func (it ImageType) Name() string { + switch it { + case TypeHelm: + return "Helm Chart" + default: + return "OCI / Docker" + } +} + +// Metadata represents the metadata of a Container package +type Metadata struct { + Type ImageType `json:"type"` + IsTagged bool `json:"is_tagged"` + Platform string `json:"platform,omitempty"` + Description string `json:"description,omitempty"` + Authors []string `json:"authors,omitempty"` + Licenses string `json:"license,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + DocumentationURL string `json:"documentation_url,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + ImageLayers []string `json:"layer_creation,omitempty"` + MultiArch map[string]string `json:"multiarch,omitempty"` +} + +// ParseImageConfig parses the metadata of an image config +func ParseImageConfig(mediaType oci.MediaType, r io.Reader) (*Metadata, error) { + if strings.EqualFold(string(mediaType), helm.ConfigMediaType) { + return parseHelmConfig(r) + } + + // fallback to OCI Image Config + return parseOCIImageConfig(r) +} + +func parseOCIImageConfig(r io.Reader) (*Metadata, error) { + var image oci.Image + if err := json.NewDecoder(r).Decode(&image); err != nil { + return nil, err + } + + platform := DefaultPlatform + if image.OS != "" && image.Architecture != "" { + platform = fmt.Sprintf("%s/%s", image.OS, image.Architecture) + if image.Variant != "" { + platform = fmt.Sprintf("%s/%s", platform, image.Variant) + } + } + + imageLayers := make([]string, 0, len(image.History)) + for _, history := range image.History { + cmd := history.CreatedBy + if i := strings.Index(cmd, "#(nop) "); i != -1 { + cmd = strings.TrimSpace(cmd[i+7:]) + } + imageLayers = append(imageLayers, cmd) + } + + metadata := &Metadata{ + Type: TypeOCI, + Platform: platform, + Licenses: image.Config.Labels[labelLicenses], + ProjectURL: image.Config.Labels[labelURL], + RepositoryURL: image.Config.Labels[labelSource], + DocumentationURL: image.Config.Labels[labelDocumentation], + Description: image.Config.Labels[labelDescription], + Labels: image.Config.Labels, + ImageLayers: imageLayers, + } + + if authors, ok := image.Config.Labels[labelAuthors]; ok { + metadata.Authors = []string{authors} + } + + if !validation.IsValidURL(metadata.ProjectURL) { + metadata.ProjectURL = "" + } + if !validation.IsValidURL(metadata.RepositoryURL) { + metadata.RepositoryURL = "" + } + if !validation.IsValidURL(metadata.DocumentationURL) { + metadata.DocumentationURL = "" + } + + return metadata, nil +} + +func parseHelmConfig(r io.Reader) (*Metadata, error) { + var config helm.Metadata + if err := json.NewDecoder(r).Decode(&config); err != nil { + return nil, err + } + + metadata := &Metadata{ + Type: TypeHelm, + Description: config.Description, + ProjectURL: config.Home, + } + + if len(config.Maintainers) > 0 { + authors := make([]string, 0, len(config.Maintainers)) + for _, maintainer := range config.Maintainers { + authors = append(authors, maintainer.Name) + } + metadata.Authors = authors + } + + if len(config.Sources) > 0 && validation.IsValidURL(config.Sources[0]) { + metadata.RepositoryURL = config.Sources[0] + } + if !validation.IsValidURL(metadata.ProjectURL) { + metadata.ProjectURL = "" + } + + return metadata, nil +} diff --git a/modules/packages/container/metadata_test.go b/modules/packages/container/metadata_test.go new file mode 100644 index 0000000000..9400cf6954 --- /dev/null +++ b/modules/packages/container/metadata_test.go @@ -0,0 +1,62 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package container + +import ( + "strings" + "testing" + + "code.gitea.io/gitea/modules/packages/container/helm" + "code.gitea.io/gitea/modules/packages/container/oci" + + "github.com/stretchr/testify/assert" +) + +func TestParseImageConfig(t *testing.T) { + description := "Image Description" + author := "Gitea" + license := "MIT" + projectURL := "https://gitea.io" + repositoryURL := "https://gitea.com/gitea" + documentationURL := "https://docs.gitea.io" + + configOCI := `{"config": {"labels": {"` + labelAuthors + `": "` + author + `", "` + labelLicenses + `": "` + license + `", "` + labelURL + `": "` + projectURL + `", "` + labelSource + `": "` + repositoryURL + `", "` + labelDocumentation + `": "` + documentationURL + `", "` + labelDescription + `": "` + description + `"}}, "history": [{"created_by": "do it 1"}, {"created_by": "dummy #(nop) do it 2"}]}` + + metadata, err := ParseImageConfig(oci.MediaType(oci.MediaTypeImageManifest), strings.NewReader(configOCI)) + assert.NoError(t, err) + + assert.Equal(t, TypeOCI, metadata.Type) + assert.Equal(t, description, metadata.Description) + assert.ElementsMatch(t, []string{author}, metadata.Authors) + assert.Equal(t, license, metadata.Licenses) + assert.Equal(t, projectURL, metadata.ProjectURL) + assert.Equal(t, repositoryURL, metadata.RepositoryURL) + assert.Equal(t, documentationURL, metadata.DocumentationURL) + assert.Equal(t, []string{"do it 1", "do it 2"}, metadata.ImageLayers) + assert.Equal( + t, + map[string]string{ + labelAuthors: author, + labelLicenses: license, + labelURL: projectURL, + labelSource: repositoryURL, + labelDocumentation: documentationURL, + labelDescription: description, + }, + metadata.Labels, + ) + assert.Empty(t, metadata.MultiArch) + + configHelm := `{"description":"` + description + `", "home": "` + projectURL + `", "sources": ["` + repositoryURL + `"], "maintainers":[{"name":"` + author + `"}]}` + + metadata, err = ParseImageConfig(oci.MediaType(helm.ConfigMediaType), strings.NewReader(configHelm)) + assert.NoError(t, err) + + assert.Equal(t, TypeHelm, metadata.Type) + assert.Equal(t, description, metadata.Description) + assert.ElementsMatch(t, []string{author}, metadata.Authors) + assert.Equal(t, projectURL, metadata.ProjectURL) + assert.Equal(t, repositoryURL, metadata.RepositoryURL) +} diff --git a/modules/packages/container/oci/digest.go b/modules/packages/container/oci/digest.go new file mode 100644 index 0000000000..5234814cfe --- /dev/null +++ b/modules/packages/container/oci/digest.go @@ -0,0 +1,27 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package oci + +import ( + "regexp" + "strings" +) + +var digestPattern = regexp.MustCompile(`\Asha256:[a-f0-9]{64}\z`) + +type Digest string + +// Validate checks if the digest has a valid SHA256 signature +func (d Digest) Validate() bool { + return digestPattern.MatchString(string(d)) +} + +func (d Digest) Hash() string { + p := strings.SplitN(string(d), ":", 2) + if len(p) != 2 { + return "" + } + return p[1] +} diff --git a/modules/packages/container/oci/mediatype.go b/modules/packages/container/oci/mediatype.go new file mode 100644 index 0000000000..2636fbe288 --- /dev/null +++ b/modules/packages/container/oci/mediatype.go @@ -0,0 +1,36 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package oci + +import ( + "strings" +) + +const ( + MediaTypeImageManifest = "application/vnd.oci.image.manifest.v1+json" + MediaTypeImageIndex = "application/vnd.oci.image.index.v1+json" + MediaTypeDockerManifest = "application/vnd.docker.distribution.manifest.v2+json" + MediaTypeDockerManifestList = "application/vnd.docker.distribution.manifest.list.v2+json" +) + +type MediaType string + +// IsValid tests if the media type is in the OCI or Docker namespace +func (m MediaType) IsValid() bool { + s := string(m) + return strings.HasPrefix(s, "application/vnd.docker.") || strings.HasPrefix(s, "application/vnd.oci.") +} + +// IsImageManifest tests if the media type is an image manifest +func (m MediaType) IsImageManifest() bool { + s := string(m) + return strings.EqualFold(s, MediaTypeDockerManifest) || strings.EqualFold(s, MediaTypeImageManifest) +} + +// IsImageIndex tests if the media type is an image index +func (m MediaType) IsImageIndex() bool { + s := string(m) + return strings.EqualFold(s, MediaTypeDockerManifestList) || strings.EqualFold(s, MediaTypeImageIndex) +} diff --git a/modules/packages/container/oci/oci.go b/modules/packages/container/oci/oci.go new file mode 100644 index 0000000000..01cca8fe69 --- /dev/null +++ b/modules/packages/container/oci/oci.go @@ -0,0 +1,191 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package oci + +import ( + "time" +) + +// https://github.com/opencontainers/image-spec/tree/main/specs-go/v1 + +// ImageConfig defines the execution parameters which should be used as a base when running a container using an image. +type ImageConfig struct { + // User defines the username or UID which the process in the container should run as. + User string `json:"User,omitempty"` + + // ExposedPorts a set of ports to expose from a container running this image. + ExposedPorts map[string]struct{} `json:"ExposedPorts,omitempty"` + + // Env is a list of environment variables to be used in a container. + Env []string `json:"Env,omitempty"` + + // Entrypoint defines a list of arguments to use as the command to execute when the container starts. + Entrypoint []string `json:"Entrypoint,omitempty"` + + // Cmd defines the default arguments to the entrypoint of the container. + Cmd []string `json:"Cmd,omitempty"` + + // Volumes is a set of directories describing where the process is likely write data specific to a container instance. + Volumes map[string]struct{} `json:"Volumes,omitempty"` + + // WorkingDir sets the current working directory of the entrypoint process in the container. + WorkingDir string `json:"WorkingDir,omitempty"` + + // Labels contains arbitrary metadata for the container. + Labels map[string]string `json:"Labels,omitempty"` + + // StopSignal contains the system call signal that will be sent to the container to exit. + StopSignal string `json:"StopSignal,omitempty"` +} + +// RootFS describes a layer content addresses +type RootFS struct { + // Type is the type of the rootfs. + Type string `json:"type"` + + // DiffIDs is an array of layer content hashes, in order from bottom-most to top-most. + DiffIDs []string `json:"diff_ids"` +} + +// History describes the history of a layer. +type History struct { + // Created is the combined date and time at which the layer was created, formatted as defined by RFC 3339, section 5.6. + Created *time.Time `json:"created,omitempty"` + + // CreatedBy is the command which created the layer. + CreatedBy string `json:"created_by,omitempty"` + + // Author is the author of the build point. + Author string `json:"author,omitempty"` + + // Comment is a custom message set when creating the layer. + Comment string `json:"comment,omitempty"` + + // EmptyLayer is used to mark if the history item created a filesystem diff. + EmptyLayer bool `json:"empty_layer,omitempty"` +} + +// Image is the JSON structure which describes some basic information about the image. +// This provides the `application/vnd.oci.image.config.v1+json` mediatype when marshalled to JSON. +type Image struct { + // Created is the combined date and time at which the image was created, formatted as defined by RFC 3339, section 5.6. + Created *time.Time `json:"created,omitempty"` + + // Author defines the name and/or email address of the person or entity which created and is responsible for maintaining the image. + Author string `json:"author,omitempty"` + + // Architecture is the CPU architecture which the binaries in this image are built to run on. + Architecture string `json:"architecture"` + + // Variant is the variant of the specified CPU architecture which image binaries are intended to run on. + Variant string `json:"variant,omitempty"` + + // OS is the name of the operating system which the image is built to run on. + OS string `json:"os"` + + // OSVersion is an optional field specifying the operating system + // version, for example on Windows `10.0.14393.1066`. + OSVersion string `json:"os.version,omitempty"` + + // OSFeatures is an optional field specifying an array of strings, + // each listing a required OS feature (for example on Windows `win32k`). + OSFeatures []string `json:"os.features,omitempty"` + + // Config defines the execution parameters which should be used as a base when running a container using the image. + Config ImageConfig `json:"config,omitempty"` + + // RootFS references the layer content addresses used by the image. + RootFS RootFS `json:"rootfs"` + + // History describes the history of each layer. + History []History `json:"history,omitempty"` +} + +// Descriptor describes the disposition of targeted content. +// This structure provides `application/vnd.oci.descriptor.v1+json` mediatype +// when marshalled to JSON. +type Descriptor struct { + // MediaType is the media type of the object this schema refers to. + MediaType MediaType `json:"mediaType,omitempty"` + + // Digest is the digest of the targeted content. + Digest Digest `json:"digest"` + + // Size specifies the size in bytes of the blob. + Size int64 `json:"size"` + + // URLs specifies a list of URLs from which this object MAY be downloaded + URLs []string `json:"urls,omitempty"` + + // Annotations contains arbitrary metadata relating to the targeted content. + Annotations map[string]string `json:"annotations,omitempty"` + + // Data is an embedding of the targeted content. This is encoded as a base64 + // string when marshalled to JSON (automatically, by encoding/json). If + // present, Data can be used directly to avoid fetching the targeted content. + Data []byte `json:"data,omitempty"` + + // Platform describes the platform which the image in the manifest runs on. + // + // This should only be used when referring to a manifest. + Platform *Platform `json:"platform,omitempty"` +} + +// Platform describes the platform which the image in the manifest runs on. +type Platform struct { + // Architecture field specifies the CPU architecture, for example + // `amd64` or `ppc64`. + Architecture string `json:"architecture"` + + // OS specifies the operating system, for example `linux` or `windows`. + OS string `json:"os"` + + // OSVersion is an optional field specifying the operating system + // version, for example on Windows `10.0.14393.1066`. + OSVersion string `json:"os.version,omitempty"` + + // OSFeatures is an optional field specifying an array of strings, + // each listing a required OS feature (for example on Windows `win32k`). + OSFeatures []string `json:"os.features,omitempty"` + + // Variant is an optional field specifying a variant of the CPU, for + // example `v7` to specify ARMv7 when architecture is `arm`. + Variant string `json:"variant,omitempty"` +} + +type SchemaMediaBase struct { + // SchemaVersion is the image manifest schema that this image follows + SchemaVersion int `json:"schemaVersion"` + + // MediaType specifies the type of this document data structure e.g. `application/vnd.oci.image.manifest.v1+json` + MediaType MediaType `json:"mediaType,omitempty"` +} + +// Manifest provides `application/vnd.oci.image.manifest.v1+json` mediatype structure when marshalled to JSON. +type Manifest struct { + SchemaMediaBase + + // Config references a configuration object for a container, by digest. + // The referenced configuration object is a JSON blob that the runtime uses to set up the container. + Config Descriptor `json:"config"` + + // Layers is an indexed list of layers referenced by the manifest. + Layers []Descriptor `json:"layers"` + + // Annotations contains arbitrary metadata for the image manifest. + Annotations map[string]string `json:"annotations,omitempty"` +} + +// Index references manifests for various platforms. +// This structure provides `application/vnd.oci.image.index.v1+json` mediatype when marshalled to JSON. +type Index struct { + SchemaMediaBase + + // Manifests references platform specific manifests. + Manifests []Descriptor `json:"manifests"` + + // Annotations contains arbitrary metadata for the image index. + Annotations map[string]string `json:"annotations,omitempty"` +} diff --git a/modules/packages/container/oci/reference.go b/modules/packages/container/oci/reference.go new file mode 100644 index 0000000000..120ff122d4 --- /dev/null +++ b/modules/packages/container/oci/reference.go @@ -0,0 +1,17 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package oci + +import ( + "regexp" +) + +var referencePattern = regexp.MustCompile(`\A[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}\z`) + +type Reference string + +func (r Reference) Validate() bool { + return referencePattern.MatchString(string(r)) +} diff --git a/modules/packages/content_store.go b/modules/packages/content_store.go new file mode 100644 index 0000000000..64c3eedc23 --- /dev/null +++ b/modules/packages/content_store.go @@ -0,0 +1,47 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "io" + "path" + + "code.gitea.io/gitea/modules/storage" +) + +// BlobHash256Key is the key to address a blob content +type BlobHash256Key string + +// ContentStore is a wrapper around ObjectStorage +type ContentStore struct { + store storage.ObjectStorage +} + +// NewContentStore creates the default package store +func NewContentStore() *ContentStore { + contentStore := &ContentStore{storage.Packages} + return contentStore +} + +// Get gets a package blob +func (s *ContentStore) Get(key BlobHash256Key) (storage.Object, error) { + return s.store.Open(keyToRelativePath(key)) +} + +// Save stores a package blob +func (s *ContentStore) Save(key BlobHash256Key, r io.Reader, size int64) error { + _, err := s.store.Save(keyToRelativePath(key), r, size) + return err +} + +// Delete deletes a package blob +func (s *ContentStore) Delete(key BlobHash256Key) error { + return s.store.Delete(keyToRelativePath(key)) +} + +// keyToRelativePath converts the sha256 key aabb000000... to aa/bb/aabb000000... +func keyToRelativePath(key BlobHash256Key) string { + return path.Join(string(key)[0:2], string(key)[2:4], string(key)) +} diff --git a/modules/packages/hashed_buffer.go b/modules/packages/hashed_buffer.go new file mode 100644 index 0000000000..3f8cafcfb5 --- /dev/null +++ b/modules/packages/hashed_buffer.go @@ -0,0 +1,70 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "io" + + "code.gitea.io/gitea/modules/util/filebuffer" +) + +// HashedSizeReader provide methods to read, sum hashes and a Size method +type HashedSizeReader interface { + io.Reader + HashSummer + Size() int64 +} + +// HashedBuffer is buffer which calculates multiple checksums +type HashedBuffer struct { + *filebuffer.FileBackedBuffer + + hash *MultiHasher + + combinedWriter io.Writer +} + +// NewHashedBuffer creates a hashed buffer with a specific maximum memory size +func NewHashedBuffer(maxMemorySize int) (*HashedBuffer, error) { + b, err := filebuffer.New(maxMemorySize) + if err != nil { + return nil, err + } + + hash := NewMultiHasher() + + combinedWriter := io.MultiWriter(b, hash) + + return &HashedBuffer{ + b, + hash, + combinedWriter, + }, nil +} + +// CreateHashedBufferFromReader creates a hashed buffer and copies the provided reader data into it. +func CreateHashedBufferFromReader(r io.Reader, maxMemorySize int) (*HashedBuffer, error) { + b, err := NewHashedBuffer(maxMemorySize) + if err != nil { + return nil, err + } + + _, err = io.Copy(b, r) + if err != nil { + return nil, err + } + + return b, nil +} + +// Write implements io.Writer +func (b *HashedBuffer) Write(p []byte) (int, error) { + return b.combinedWriter.Write(p) +} + +// Sums gets the MD5, SHA1, SHA256 and SHA512 checksums of the data +func (b *HashedBuffer) Sums() (hashMD5, hashSHA1, hashSHA256, hashSHA512 []byte) { + return b.hash.Sums() +} diff --git a/modules/packages/maven/metadata.go b/modules/packages/maven/metadata.go new file mode 100644 index 0000000000..6ee9d69687 --- /dev/null +++ b/modules/packages/maven/metadata.go @@ -0,0 +1,89 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package maven + +import ( + "encoding/xml" + "io" + + "code.gitea.io/gitea/modules/validation" +) + +// Metadata represents the metadata of a Maven package +type Metadata struct { + GroupID string `json:"group_id,omitempty"` + ArtifactID string `json:"artifact_id,omitempty"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + Licenses []string `json:"licenses,omitempty"` + Dependencies []*Dependency `json:"dependencies,omitempty"` +} + +// Dependency represents a dependency of a Maven package +type Dependency struct { + GroupID string `json:"group_id,omitempty"` + ArtifactID string `json:"artifact_id,omitempty"` + Version string `json:"version,omitempty"` +} + +type pomStruct struct { + XMLName xml.Name `xml:"project"` + GroupID string `xml:"groupId"` + ArtifactID string `xml:"artifactId"` + Version string `xml:"version"` + Name string `xml:"name"` + Description string `xml:"description"` + URL string `xml:"url"` + Licenses []struct { + Name string `xml:"name"` + URL string `xml:"url"` + Distribution string `xml:"distribution"` + } `xml:"licenses>license"` + Dependencies []struct { + GroupID string `xml:"groupId"` + ArtifactID string `xml:"artifactId"` + Version string `xml:"version"` + Scope string `xml:"scope"` + } `xml:"dependencies>dependency"` +} + +// ParsePackageMetaData parses the metadata of a pom file +func ParsePackageMetaData(r io.Reader) (*Metadata, error) { + var pom pomStruct + if err := xml.NewDecoder(r).Decode(&pom); err != nil { + return nil, err + } + + if !validation.IsValidURL(pom.URL) { + pom.URL = "" + } + + licenses := make([]string, 0, len(pom.Licenses)) + for _, l := range pom.Licenses { + if l.Name != "" { + licenses = append(licenses, l.Name) + } + } + + dependencies := make([]*Dependency, 0, len(pom.Dependencies)) + for _, d := range pom.Dependencies { + dependencies = append(dependencies, &Dependency{ + GroupID: d.GroupID, + ArtifactID: d.ArtifactID, + Version: d.Version, + }) + } + + return &Metadata{ + GroupID: pom.GroupID, + ArtifactID: pom.ArtifactID, + Name: pom.Name, + Description: pom.Description, + ProjectURL: pom.URL, + Licenses: licenses, + Dependencies: dependencies, + }, nil +} diff --git a/modules/packages/maven/metadata_test.go b/modules/packages/maven/metadata_test.go new file mode 100644 index 0000000000..a17d456560 --- /dev/null +++ b/modules/packages/maven/metadata_test.go @@ -0,0 +1,73 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package maven + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + groupID = "org.gitea" + artifactID = "my-project" + version = "1.0.1" + name = "My Gitea Project" + description = "Package Description" + projectURL = "https://gitea.io" + license = "MIT" + dependencyGroupID = "org.gitea.core" + dependencyArtifactID = "git" + dependencyVersion = "5.0.0" +) + +const pomContent = `<?xml version="1.0"?> +<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <groupId>` + groupID + `</groupId> + <artifactId>` + artifactID + `</artifactId> + <version>` + version + `</version> + <name>` + name + `</name> + <description>` + description + `</description> + <url>` + projectURL + `</url> + <licenses> + <license> + <name>` + license + `</name> + </license> + </licenses> + <dependencies> + <dependency> + <groupId>` + dependencyGroupID + `</groupId> + <artifactId>` + dependencyArtifactID + `</artifactId> + <version>` + dependencyVersion + `</version> + </dependency> + </dependencies> +</project>` + +func TestParsePackageMetaData(t *testing.T) { + t.Run("InvalidFile", func(t *testing.T) { + m, err := ParsePackageMetaData(strings.NewReader("")) + assert.Nil(t, m) + assert.Error(t, err) + }) + + t.Run("Valid", func(t *testing.T) { + m, err := ParsePackageMetaData(strings.NewReader(pomContent)) + assert.NoError(t, err) + assert.NotNil(t, m) + + assert.Equal(t, groupID, m.GroupID) + assert.Equal(t, artifactID, m.ArtifactID) + assert.Equal(t, name, m.Name) + assert.Equal(t, description, m.Description) + assert.Equal(t, projectURL, m.ProjectURL) + assert.Len(t, m.Licenses, 1) + assert.Equal(t, license, m.Licenses[0]) + assert.Len(t, m.Dependencies, 1) + assert.Equal(t, dependencyGroupID, m.Dependencies[0].GroupID) + assert.Equal(t, dependencyArtifactID, m.Dependencies[0].ArtifactID) + assert.Equal(t, dependencyVersion, m.Dependencies[0].Version) + }) +} diff --git a/modules/packages/multi_hasher.go b/modules/packages/multi_hasher.go new file mode 100644 index 0000000000..0659a18d2a --- /dev/null +++ b/modules/packages/multi_hasher.go @@ -0,0 +1,123 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding" + "errors" + "hash" + "io" +) + +const ( + marshaledSizeMD5 = 92 + marshaledSizeSHA1 = 96 + marshaledSizeSHA256 = 108 + marshaledSizeSHA512 = 204 + + marshaledSize = marshaledSizeMD5 + marshaledSizeSHA1 + marshaledSizeSHA256 + marshaledSizeSHA512 +) + +// HashSummer provide a Sums method +type HashSummer interface { + Sums() (hashMD5, hashSHA1, hashSHA256, hashSHA512 []byte) +} + +// MultiHasher calculates multiple checksums +type MultiHasher struct { + md5 hash.Hash + sha1 hash.Hash + sha256 hash.Hash + sha512 hash.Hash + + combinedWriter io.Writer +} + +// NewMultiHasher creates a multi hasher +func NewMultiHasher() *MultiHasher { + md5 := md5.New() + sha1 := sha1.New() + sha256 := sha256.New() + sha512 := sha512.New() + + combinedWriter := io.MultiWriter(md5, sha1, sha256, sha512) + + return &MultiHasher{ + md5, + sha1, + sha256, + sha512, + combinedWriter, + } +} + +// MarshalBinary implements encoding.BinaryMarshaler +func (h *MultiHasher) MarshalBinary() ([]byte, error) { + md5Bytes, err := h.md5.(encoding.BinaryMarshaler).MarshalBinary() + if err != nil { + return nil, err + } + sha1Bytes, err := h.sha1.(encoding.BinaryMarshaler).MarshalBinary() + if err != nil { + return nil, err + } + sha256Bytes, err := h.sha256.(encoding.BinaryMarshaler).MarshalBinary() + if err != nil { + return nil, err + } + sha512Bytes, err := h.sha512.(encoding.BinaryMarshaler).MarshalBinary() + if err != nil { + return nil, err + } + + b := make([]byte, 0, marshaledSize) + b = append(b, md5Bytes...) + b = append(b, sha1Bytes...) + b = append(b, sha256Bytes...) + b = append(b, sha512Bytes...) + return b, nil +} + +// UnmarshalBinary implements encoding.BinaryUnmarshaler +func (h *MultiHasher) UnmarshalBinary(b []byte) error { + if len(b) != marshaledSize { + return errors.New("invalid hash state size") + } + + if err := h.md5.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[:marshaledSizeMD5]); err != nil { + return err + } + + b = b[marshaledSizeMD5:] + if err := h.sha1.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[:marshaledSizeSHA1]); err != nil { + return err + } + + b = b[marshaledSizeSHA1:] + if err := h.sha256.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[:marshaledSizeSHA256]); err != nil { + return err + } + + b = b[marshaledSizeSHA256:] + return h.sha512.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[:marshaledSizeSHA512]) +} + +// Write implements io.Writer +func (h *MultiHasher) Write(p []byte) (int, error) { + return h.combinedWriter.Write(p) +} + +// Sums gets the MD5, SHA1, SHA256 and SHA512 checksums of the data +func (h *MultiHasher) Sums() (hashMD5, hashSHA1, hashSHA256, hashSHA512 []byte) { + hashMD5 = h.md5.Sum(nil) + hashSHA1 = h.sha1.Sum(nil) + hashSHA256 = h.sha256.Sum(nil) + hashSHA512 = h.sha512.Sum(nil) + return +} diff --git a/modules/packages/multi_hasher_test.go b/modules/packages/multi_hasher_test.go new file mode 100644 index 0000000000..6c895ce120 --- /dev/null +++ b/modules/packages/multi_hasher_test.go @@ -0,0 +1,54 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + expectedMD5 = "e3bef03c5f3b7f6b3ab3e3053ed71e9c" + expectedSHA1 = "060b3b99f88e96085b4a68e095bc9e3d1d91e1bc" + expectedSHA256 = "6ccce4863b70f258d691f59609d31b4502e1ba5199942d3bc5d35d17a4ce771d" + expectedSHA512 = "7f70e439ba8c52025c1f06cdf6ae443c4b8ed2e90059cdb9bbbf8adf80846f185a24acca9245b128b226d61753b0d7ed46580a69c8999eeff3bc13a4d0bd816c" +) + +func TestMultiHasherSums(t *testing.T) { + t.Run("Sums", func(t *testing.T) { + h := NewMultiHasher() + h.Write([]byte("gitea")) + + hashMD5, hashSHA1, hashSHA256, hashSHA512 := h.Sums() + + assert.Equal(t, expectedMD5, fmt.Sprintf("%x", hashMD5)) + assert.Equal(t, expectedSHA1, fmt.Sprintf("%x", hashSHA1)) + assert.Equal(t, expectedSHA256, fmt.Sprintf("%x", hashSHA256)) + assert.Equal(t, expectedSHA512, fmt.Sprintf("%x", hashSHA512)) + }) + + t.Run("State", func(t *testing.T) { + h := NewMultiHasher() + h.Write([]byte("git")) + + state, err := h.MarshalBinary() + assert.NoError(t, err) + + h2 := NewMultiHasher() + err = h2.UnmarshalBinary(state) + assert.NoError(t, err) + + h2.Write([]byte("ea")) + + hashMD5, hashSHA1, hashSHA256, hashSHA512 := h2.Sums() + + assert.Equal(t, expectedMD5, fmt.Sprintf("%x", hashMD5)) + assert.Equal(t, expectedSHA1, fmt.Sprintf("%x", hashSHA1)) + assert.Equal(t, expectedSHA256, fmt.Sprintf("%x", hashSHA256)) + assert.Equal(t, expectedSHA512, fmt.Sprintf("%x", hashSHA512)) + }) +} diff --git a/modules/packages/npm/creator.go b/modules/packages/npm/creator.go new file mode 100644 index 0000000000..88ce55ecdb --- /dev/null +++ b/modules/packages/npm/creator.go @@ -0,0 +1,256 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package npm + +import ( + "bytes" + "crypto/sha1" + "crypto/sha512" + "encoding/base64" + "errors" + "fmt" + "io" + "regexp" + "strings" + "time" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/validation" + + "github.com/hashicorp/go-version" +) + +var ( + // ErrInvalidPackage indicates an invalid package + ErrInvalidPackage = errors.New("The package is invalid") + // ErrInvalidPackageName indicates an invalid name + ErrInvalidPackageName = errors.New("The package name is invalid") + // ErrInvalidPackageVersion indicates an invalid version + ErrInvalidPackageVersion = errors.New("The package version is invalid") + // ErrInvalidAttachment indicates a invalid attachment + ErrInvalidAttachment = errors.New("The package attachment is invalid") + // ErrInvalidIntegrity indicates an integrity validation error + ErrInvalidIntegrity = errors.New("Failed to validate integrity") +) + +var nameMatch = regexp.MustCompile(`\A((@[^\s\/~'!\(\)\*]+?)[\/])?([^_.][^\s\/~'!\(\)\*]+)\z`) + +// Package represents a npm package +type Package struct { + Name string + Version string + DistTags []string + Metadata Metadata + Filename string + Data []byte +} + +// PackageMetadata https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#package +type PackageMetadata struct { + ID string `json:"_id"` + Name string `json:"name"` + Description string `json:"description"` + DistTags map[string]string `json:"dist-tags,omitempty"` + Versions map[string]*PackageMetadataVersion `json:"versions"` + Readme string `json:"readme,omitempty"` + Maintainers []User `json:"maintainers,omitempty"` + Time map[string]time.Time `json:"time,omitempty"` + Homepage string `json:"homepage,omitempty"` + Keywords []string `json:"keywords,omitempty"` + Repository Repository `json:"repository,omitempty"` + Author User `json:"author"` + ReadmeFilename string `json:"readmeFilename,omitempty"` + Users map[string]bool `json:"users,omitempty"` + License string `json:"license,omitempty"` +} + +// PackageMetadataVersion https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version +type PackageMetadataVersion struct { + ID string `json:"_id"` + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Author User `json:"author"` + Homepage string `json:"homepage,omitempty"` + License string `json:"license,omitempty"` + Repository Repository `json:"repository,omitempty"` + Keywords []string `json:"keywords,omitempty"` + Dependencies map[string]string `json:"dependencies,omitempty"` + DevDependencies map[string]string `json:"devDependencies,omitempty"` + PeerDependencies map[string]string `json:"peerDependencies,omitempty"` + OptionalDependencies map[string]string `json:"optionalDependencies,omitempty"` + Readme string `json:"readme,omitempty"` + Dist PackageDistribution `json:"dist"` + Maintainers []User `json:"maintainers,omitempty"` +} + +// PackageDistribution https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version +type PackageDistribution struct { + Integrity string `json:"integrity"` + Shasum string `json:"shasum"` + Tarball string `json:"tarball"` + FileCount int `json:"fileCount,omitempty"` + UnpackedSize int `json:"unpackedSize,omitempty"` + NpmSignature string `json:"npm-signature,omitempty"` +} + +// User https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#package +type User struct { + Username string `json:"username,omitempty"` + Name string `json:"name"` + Email string `json:"email,omitempty"` + URL string `json:"url,omitempty"` +} + +// UnmarshalJSON is needed because User objects can be strings or objects +func (u *User) UnmarshalJSON(data []byte) error { + switch data[0] { + case '"': + if err := json.Unmarshal(data, &u.Name); err != nil { + return err + } + case '{': + var tmp struct { + Username string `json:"username"` + Name string `json:"name"` + Email string `json:"email"` + URL string `json:"url"` + } + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + u.Username = tmp.Username + u.Name = tmp.Name + u.Email = tmp.Email + u.URL = tmp.URL + } + return nil +} + +// Repository https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version +type Repository struct { + Type string `json:"type"` + URL string `json:"url"` +} + +// PackageAttachment https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#package +type PackageAttachment struct { + ContentType string `json:"content_type"` + Data string `json:"data"` + Length int `json:"length"` +} + +type packageUpload struct { + PackageMetadata + Attachments map[string]*PackageAttachment `json:"_attachments"` +} + +// ParsePackage parses the content into a npm package +func ParsePackage(r io.Reader) (*Package, error) { + var upload packageUpload + if err := json.NewDecoder(r).Decode(&upload); err != nil { + return nil, err + } + + for _, meta := range upload.Versions { + if !validateName(meta.Name) { + return nil, ErrInvalidPackageName + } + + v, err := version.NewSemver(meta.Version) + if err != nil { + return nil, ErrInvalidPackageVersion + } + + scope := "" + name := meta.Name + nameParts := strings.SplitN(meta.Name, "/", 2) + if len(nameParts) == 2 { + scope = nameParts[0] + name = nameParts[1] + } + + if !validation.IsValidURL(meta.Homepage) { + meta.Homepage = "" + } + + p := &Package{ + Name: meta.Name, + Version: v.String(), + DistTags: make([]string, 0, 1), + Metadata: Metadata{ + Scope: scope, + Name: name, + Description: meta.Description, + Author: meta.Author.Name, + License: meta.License, + ProjectURL: meta.Homepage, + Keywords: meta.Keywords, + Dependencies: meta.Dependencies, + DevelopmentDependencies: meta.DevDependencies, + PeerDependencies: meta.PeerDependencies, + OptionalDependencies: meta.OptionalDependencies, + Readme: meta.Readme, + }, + } + + for tag := range upload.DistTags { + p.DistTags = append(p.DistTags, tag) + } + + p.Filename = strings.ToLower(fmt.Sprintf("%s-%s.tgz", name, p.Version)) + + attachment := func() *PackageAttachment { + for _, a := range upload.Attachments { + return a + } + return nil + }() + if attachment == nil || len(attachment.Data) == 0 { + return nil, ErrInvalidAttachment + } + + data, err := base64.StdEncoding.DecodeString(attachment.Data) + if err != nil { + return nil, ErrInvalidAttachment + } + p.Data = data + + integrity := strings.SplitN(meta.Dist.Integrity, "-", 2) + if len(integrity) != 2 { + return nil, ErrInvalidIntegrity + } + integrityHash, err := base64.StdEncoding.DecodeString(integrity[1]) + if err != nil { + return nil, ErrInvalidIntegrity + } + var hash []byte + switch integrity[0] { + case "sha1": + tmp := sha1.Sum(data) + hash = tmp[:] + case "sha512": + tmp := sha512.Sum512(data) + hash = tmp[:] + } + if !bytes.Equal(integrityHash, hash) { + return nil, ErrInvalidIntegrity + } + + return p, nil + } + + return nil, ErrInvalidPackage +} + +func validateName(name string) bool { + if strings.TrimSpace(name) != name { + return false + } + if len(name) == 0 || len(name) > 214 { + return false + } + return nameMatch.MatchString(name) +} diff --git a/modules/packages/npm/creator_test.go b/modules/packages/npm/creator_test.go new file mode 100644 index 0000000000..64ae6238f3 --- /dev/null +++ b/modules/packages/npm/creator_test.go @@ -0,0 +1,272 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package npm + +import ( + "bytes" + "encoding/base64" + "fmt" + "strings" + "testing" + + "code.gitea.io/gitea/modules/json" + + "github.com/stretchr/testify/assert" +) + +func TestParsePackage(t *testing.T) { + packageScope := "@scope" + packageName := "test-package" + packageFullName := packageScope + "/" + packageName + packageVersion := "1.0.1-pre" + packageTag := "latest" + packageAuthor := "KN4CK3R" + packageDescription := "Test Description" + data := "H4sIAAAAAAAA/ytITM5OTE/VL4DQelnF+XkMVAYGBgZmJiYK2MRBwNDcSIHB2NTMwNDQzMwAqA7IMDUxA9LUdgg2UFpcklgEdAql5kD8ogCnhwio5lJQUMpLzE1VslJQcihOzi9I1S9JLS7RhSYIJR2QgrLUouLM/DyQGkM9Az1D3YIiqExKanFyUWZBCVQ2BKhVwQVJDKwosbQkI78IJO/tZ+LsbRykxFXLNdA+HwWjYBSMgpENACgAbtAACAAA" + integrity := "sha512-yA4FJsVhetynGfOC1jFf79BuS+jrHbm0fhh+aHzCQkOaOBXKf9oBnC4a6DnLLnEsHQDRLYd00cwj8sCXpC+wIg==" + + t.Run("InvalidUpload", func(t *testing.T) { + p, err := ParsePackage(bytes.NewReader([]byte{0})) + assert.Nil(t, p) + assert.Error(t, err) + }) + + t.Run("InvalidUploadNoData", func(t *testing.T) { + b, _ := json.Marshal(packageUpload{}) + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidPackage) + }) + + t.Run("InvalidPackageName", func(t *testing.T) { + test := func(t *testing.T, name string) { + b, _ := json.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: name, + Name: name, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: name, + }, + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidPackageName) + } + + test(t, " test ") + test(t, " test") + test(t, "test ") + test(t, "te st") + test(t, "invalid/scope") + test(t, "@invalid/_name") + test(t, "@invalid/.name") + }) + + t.Run("ValidPackageName", func(t *testing.T) { + test := func(t *testing.T, name string) { + b, _ := json.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: name, + Name: name, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: name, + }, + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidPackageVersion) + } + + test(t, "test") + test(t, "@scope/name") + test(t, packageFullName) + }) + + t.Run("InvalidPackageVersion", func(t *testing.T) { + version := "first-version" + b, _ := json.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: packageFullName, + Name: packageFullName, + Versions: map[string]*PackageMetadataVersion{ + version: { + Name: packageFullName, + Version: version, + }, + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidPackageVersion) + }) + + t.Run("InvalidAttachment", func(t *testing.T) { + b, _ := json.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: packageFullName, + Name: packageFullName, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: packageFullName, + Version: packageVersion, + }, + }, + }, + Attachments: map[string]*PackageAttachment{ + "dummy.tgz": {}, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidAttachment) + }) + + t.Run("InvalidData", func(t *testing.T) { + filename := fmt.Sprintf("%s-%s.tgz", packageFullName, packageVersion) + b, _ := json.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: packageFullName, + Name: packageFullName, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: packageFullName, + Version: packageVersion, + }, + }, + }, + Attachments: map[string]*PackageAttachment{ + filename: { + Data: "/", + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidAttachment) + }) + + t.Run("InvalidIntegrity", func(t *testing.T) { + filename := fmt.Sprintf("%s-%s.tgz", packageFullName, packageVersion) + b, _ := json.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: packageFullName, + Name: packageFullName, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: packageFullName, + Version: packageVersion, + Dist: PackageDistribution{ + Integrity: "sha512-test==", + }, + }, + }, + }, + Attachments: map[string]*PackageAttachment{ + filename: { + Data: data, + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidIntegrity) + }) + + t.Run("InvalidIntegrity2", func(t *testing.T) { + filename := fmt.Sprintf("%s-%s.tgz", packageFullName, packageVersion) + b, _ := json.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: packageFullName, + Name: packageFullName, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: packageFullName, + Version: packageVersion, + Dist: PackageDistribution{ + Integrity: integrity, + }, + }, + }, + }, + Attachments: map[string]*PackageAttachment{ + filename: { + Data: base64.StdEncoding.EncodeToString([]byte("data")), + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidIntegrity) + }) + + t.Run("Valid", func(t *testing.T) { + filename := fmt.Sprintf("%s-%s.tgz", packageFullName, packageVersion) + b, _ := json.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: packageFullName, + Name: packageFullName, + DistTags: map[string]string{ + packageTag: packageVersion, + }, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: packageFullName, + Version: packageVersion, + Description: packageDescription, + Author: User{Name: packageAuthor}, + License: "MIT", + Homepage: "https://gitea.io/", + Readme: packageDescription, + Dependencies: map[string]string{ + "package": "1.2.0", + }, + Dist: PackageDistribution{ + Integrity: integrity, + }, + }, + }, + }, + Attachments: map[string]*PackageAttachment{ + filename: { + Data: data, + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.NotNil(t, p) + assert.NoError(t, err) + + assert.Equal(t, packageFullName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.Equal(t, []string{packageTag}, p.DistTags) + assert.Equal(t, fmt.Sprintf("%s-%s.tgz", strings.Split(packageFullName, "/")[1], packageVersion), p.Filename) + b, _ = base64.StdEncoding.DecodeString(data) + assert.Equal(t, b, p.Data) + assert.Equal(t, packageName, p.Metadata.Name) + assert.Equal(t, packageScope, p.Metadata.Scope) + assert.Equal(t, packageDescription, p.Metadata.Description) + assert.Equal(t, packageDescription, p.Metadata.Readme) + assert.Equal(t, packageAuthor, p.Metadata.Author) + assert.Equal(t, "MIT", p.Metadata.License) + assert.Equal(t, "https://gitea.io/", p.Metadata.ProjectURL) + assert.Contains(t, p.Metadata.Dependencies, "package") + assert.Equal(t, "1.2.0", p.Metadata.Dependencies["package"]) + }) +} diff --git a/modules/packages/npm/metadata.go b/modules/packages/npm/metadata.go new file mode 100644 index 0000000000..643a4d344b --- /dev/null +++ b/modules/packages/npm/metadata.go @@ -0,0 +1,24 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package npm + +// TagProperty is the name of the property for tag management +const TagProperty = "npm.tag" + +// Metadata represents the metadata of a npm package +type Metadata struct { + Scope string `json:"scope,omitempty"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Author string `json:"author,omitempty"` + License string `json:"license,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + Keywords []string `json:"keywords,omitempty"` + Dependencies map[string]string `json:"dependencies,omitempty"` + DevelopmentDependencies map[string]string `json:"development_dependencies,omitempty"` + PeerDependencies map[string]string `json:"peer_dependencies,omitempty"` + OptionalDependencies map[string]string `json:"optional_dependencies,omitempty"` + Readme string `json:"readme,omitempty"` +} diff --git a/modules/packages/nuget/metadata.go b/modules/packages/nuget/metadata.go new file mode 100644 index 0000000000..797bff45ac --- /dev/null +++ b/modules/packages/nuget/metadata.go @@ -0,0 +1,187 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package nuget + +import ( + "archive/zip" + "encoding/xml" + "errors" + "io" + "path/filepath" + "regexp" + "strings" + + "code.gitea.io/gitea/modules/validation" + + "github.com/hashicorp/go-version" +) + +var ( + // ErrMissingNuspecFile indicates a missing Nuspec file + ErrMissingNuspecFile = errors.New("Nuspec file is missing") + // ErrNuspecFileTooLarge indicates a Nuspec file which is too large + ErrNuspecFileTooLarge = errors.New("Nuspec file is too large") + // ErrNuspecInvalidID indicates an invalid id in the Nuspec file + ErrNuspecInvalidID = errors.New("Nuspec file contains an invalid id") + // ErrNuspecInvalidVersion indicates an invalid version in the Nuspec file + ErrNuspecInvalidVersion = errors.New("Nuspec file contains an invalid version") +) + +// PackageType specifies the package type the metadata describes +type PackageType int + +const ( + // DependencyPackage represents a package (*.nupkg) + DependencyPackage PackageType = iota + 1 + // SymbolsPackage represents a symbol package (*.snupkg) + SymbolsPackage + + PropertySymbolID = "nuget.symbol.id" +) + +var idmatch = regexp.MustCompile(`\A\w+(?:[.-]\w+)*\z`) + +const maxNuspecFileSize = 3 * 1024 * 1024 + +// Package represents a Nuget package +type Package struct { + PackageType PackageType + ID string + Version string + Metadata *Metadata +} + +// Metadata represents the metadata of a Nuget package +type Metadata struct { + Description string `json:"description,omitempty"` + ReleaseNotes string `json:"release_notes,omitempty"` + Authors string `json:"authors,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + Dependencies map[string][]Dependency `json:"dependencies,omitempty"` +} + +// Dependency represents a dependency of a Nuget package +type Dependency struct { + ID string `json:"id"` + Version string `json:"version"` +} + +type nuspecPackage struct { + Metadata struct { + ID string `xml:"id"` + Version string `xml:"version"` + Authors string `xml:"authors"` + RequireLicenseAcceptance bool `xml:"requireLicenseAcceptance"` + ProjectURL string `xml:"projectUrl"` + Description string `xml:"description"` + ReleaseNotes string `xml:"releaseNotes"` + PackageTypes struct { + PackageType []struct { + Name string `xml:"name,attr"` + } `xml:"packageType"` + } `xml:"packageTypes"` + Repository struct { + URL string `xml:"url,attr"` + } `xml:"repository"` + Dependencies struct { + Group []struct { + TargetFramework string `xml:"targetFramework,attr"` + Dependency []struct { + ID string `xml:"id,attr"` + Version string `xml:"version,attr"` + Exclude string `xml:"exclude,attr"` + } `xml:"dependency"` + } `xml:"group"` + } `xml:"dependencies"` + } `xml:"metadata"` +} + +// ParsePackageMetaData parses the metadata of a Nuget package file +func ParsePackageMetaData(r io.ReaderAt, size int64) (*Package, error) { + archive, err := zip.NewReader(r, size) + if err != nil { + return nil, err + } + + for _, file := range archive.File { + if filepath.Dir(file.Name) != "." { + continue + } + if strings.HasSuffix(strings.ToLower(file.Name), ".nuspec") { + if file.UncompressedSize64 > maxNuspecFileSize { + return nil, ErrNuspecFileTooLarge + } + f, err := archive.Open(file.Name) + if err != nil { + return nil, err + } + defer f.Close() + + return ParseNuspecMetaData(f) + } + } + return nil, ErrMissingNuspecFile +} + +// ParseNuspecMetaData parses a Nuspec file to retrieve the metadata of a Nuget package +func ParseNuspecMetaData(r io.Reader) (*Package, error) { + var p nuspecPackage + if err := xml.NewDecoder(r).Decode(&p); err != nil { + return nil, err + } + + if !idmatch.MatchString(p.Metadata.ID) { + return nil, ErrNuspecInvalidID + } + + v, err := version.NewSemver(p.Metadata.Version) + if err != nil { + return nil, ErrNuspecInvalidVersion + } + + if !validation.IsValidURL(p.Metadata.ProjectURL) { + p.Metadata.ProjectURL = "" + } + + packageType := DependencyPackage + for _, pt := range p.Metadata.PackageTypes.PackageType { + if pt.Name == "SymbolsPackage" { + packageType = SymbolsPackage + break + } + } + + m := &Metadata{ + Description: p.Metadata.Description, + ReleaseNotes: p.Metadata.ReleaseNotes, + Authors: p.Metadata.Authors, + ProjectURL: p.Metadata.ProjectURL, + RepositoryURL: p.Metadata.Repository.URL, + Dependencies: make(map[string][]Dependency), + } + + for _, group := range p.Metadata.Dependencies.Group { + deps := make([]Dependency, 0, len(group.Dependency)) + for _, dep := range group.Dependency { + if dep.ID == "" || dep.Version == "" { + continue + } + deps = append(deps, Dependency{ + ID: dep.ID, + Version: dep.Version, + }) + } + if len(deps) > 0 { + m.Dependencies[group.TargetFramework] = deps + } + } + return &Package{ + PackageType: packageType, + ID: p.Metadata.ID, + Version: v.String(), + Metadata: m, + }, nil +} diff --git a/modules/packages/nuget/metadata_test.go b/modules/packages/nuget/metadata_test.go new file mode 100644 index 0000000000..e8c7773e97 --- /dev/null +++ b/modules/packages/nuget/metadata_test.go @@ -0,0 +1,163 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package nuget + +import ( + "archive/zip" + "bytes" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + id = "System.Gitea" + semver = "1.0.1" + authors = "Gitea Authors" + projectURL = "https://gitea.io" + description = "Package Description" + releaseNotes = "Package Release Notes" + repositoryURL = "https://gitea.io/gitea/gitea" + targetFramework = ".NETStandard2.1" + dependencyID = "System.Text.Json" + dependencyVersion = "5.0.0" +) + +const nuspecContent = `<?xml version="1.0" encoding="utf-8"?> +<package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd"> + <metadata> + <id>` + id + `</id> + <version>` + semver + `</version> + <authors>` + authors + `</authors> + <requireLicenseAcceptance>true</requireLicenseAcceptance> + <projectUrl>` + projectURL + `</projectUrl> + <description>` + description + `</description> + <releaseNotes>` + releaseNotes + `</releaseNotes> + <repository url="` + repositoryURL + `" /> + <dependencies> + <group targetFramework="` + targetFramework + `"> + <dependency id="` + dependencyID + `" version="` + dependencyVersion + `" exclude="Build,Analyzers" /> + </group> + </dependencies> + </metadata> +</package>` + +const symbolsNuspecContent = `<?xml version="1.0" encoding="utf-8"?> +<package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd"> + <metadata> + <id>` + id + `</id> + <version>` + semver + `</version> + <description>` + description + `</description> + <packageTypes> + <packageType name="SymbolsPackage" /> + </packageTypes> + <dependencies> + <group targetFramework="` + targetFramework + `" /> + </dependencies> + </metadata> +</package>` + +func TestParsePackageMetaData(t *testing.T) { + createArchive := func(name, content string) []byte { + var buf bytes.Buffer + archive := zip.NewWriter(&buf) + w, _ := archive.Create(name) + w.Write([]byte(content)) + archive.Close() + return buf.Bytes() + } + + t.Run("MissingNuspecFile", func(t *testing.T) { + data := createArchive("dummy.txt", "") + + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, np) + assert.ErrorIs(t, err, ErrMissingNuspecFile) + }) + + t.Run("MissingNuspecFileInRoot", func(t *testing.T) { + data := createArchive("sub/package.nuspec", "") + + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, np) + assert.ErrorIs(t, err, ErrMissingNuspecFile) + }) + + t.Run("InvalidNuspecFile", func(t *testing.T) { + data := createArchive("package.nuspec", "") + + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, np) + assert.Error(t, err) + }) + + t.Run("InvalidPackageId", func(t *testing.T) { + data := createArchive("package.nuspec", `<?xml version="1.0" encoding="utf-8"?> + <package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd"> + <metadata></metadata> + </package>`) + + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, np) + assert.ErrorIs(t, err, ErrNuspecInvalidID) + }) + + t.Run("InvalidPackageVersion", func(t *testing.T) { + data := createArchive("package.nuspec", `<?xml version="1.0" encoding="utf-8"?> + <package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd"> + <metadata> + <id>`+id+`</id> + </metadata> + </package>`) + + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, np) + assert.ErrorIs(t, err, ErrNuspecInvalidVersion) + }) + + t.Run("Valid", func(t *testing.T) { + data := createArchive("package.nuspec", nuspecContent) + + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.NoError(t, err) + assert.NotNil(t, np) + }) +} + +func TestParseNuspecMetaData(t *testing.T) { + t.Run("Dependency Package", func(t *testing.T) { + np, err := ParseNuspecMetaData(strings.NewReader(nuspecContent)) + assert.NoError(t, err) + assert.NotNil(t, np) + assert.Equal(t, DependencyPackage, np.PackageType) + + assert.Equal(t, id, np.ID) + assert.Equal(t, semver, np.Version) + assert.Equal(t, authors, np.Metadata.Authors) + assert.Equal(t, projectURL, np.Metadata.ProjectURL) + assert.Equal(t, description, np.Metadata.Description) + assert.Equal(t, releaseNotes, np.Metadata.ReleaseNotes) + assert.Equal(t, repositoryURL, np.Metadata.RepositoryURL) + assert.Len(t, np.Metadata.Dependencies, 1) + assert.Contains(t, np.Metadata.Dependencies, targetFramework) + deps := np.Metadata.Dependencies[targetFramework] + assert.Len(t, deps, 1) + assert.Equal(t, dependencyID, deps[0].ID) + assert.Equal(t, dependencyVersion, deps[0].Version) + }) + + t.Run("Symbols Package", func(t *testing.T) { + np, err := ParseNuspecMetaData(strings.NewReader(symbolsNuspecContent)) + assert.NoError(t, err) + assert.NotNil(t, np) + assert.Equal(t, SymbolsPackage, np.PackageType) + + assert.Equal(t, id, np.ID) + assert.Equal(t, semver, np.Version) + assert.Equal(t, description, np.Metadata.Description) + assert.Empty(t, np.Metadata.Dependencies) + }) +} diff --git a/modules/packages/nuget/symbol_extractor.go b/modules/packages/nuget/symbol_extractor.go new file mode 100644 index 0000000000..13641ca6ef --- /dev/null +++ b/modules/packages/nuget/symbol_extractor.go @@ -0,0 +1,187 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package nuget + +import ( + "archive/zip" + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "path" + "path/filepath" + "strings" + + "code.gitea.io/gitea/modules/packages" +) + +var ( + ErrMissingPdbFiles = errors.New("Package does not contain PDB files") + ErrInvalidFiles = errors.New("Package contains invalid files") + ErrInvalidPdbMagicNumber = errors.New("Invalid Portable PDB magic number") + ErrMissingPdbStream = errors.New("Missing PDB stream") +) + +type PortablePdb struct { + Name string + ID string + Content *packages.HashedBuffer +} + +type PortablePdbList []*PortablePdb + +func (l PortablePdbList) Close() { + for _, pdb := range l { + pdb.Content.Close() + } +} + +// ExtractPortablePdb extracts PDB files from a .snupkg file +func ExtractPortablePdb(r io.ReaderAt, size int64) (PortablePdbList, error) { + archive, err := zip.NewReader(r, size) + if err != nil { + return nil, err + } + + var pdbs PortablePdbList + + err = func() error { + for _, file := range archive.File { + if strings.HasSuffix(file.Name, "/") { + continue + } + ext := strings.ToLower(filepath.Ext(file.Name)) + + switch ext { + case ".nuspec", ".xml", ".psmdcp", ".rels", ".p7s": + continue + case ".pdb": + f, err := archive.Open(file.Name) + if err != nil { + return err + } + + buf, err := packages.CreateHashedBufferFromReader(f, 32*1024*1024) + + f.Close() + + if err != nil { + return err + } + + id, err := ParseDebugHeaderID(buf) + if err != nil { + buf.Close() + return fmt.Errorf("Invalid PDB file: %v", err) + } + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + buf.Close() + return err + } + + pdbs = append(pdbs, &PortablePdb{ + Name: path.Base(file.Name), + ID: id, + Content: buf, + }) + default: + return ErrInvalidFiles + } + } + return nil + }() + if err != nil { + pdbs.Close() + return nil, err + } + + if len(pdbs) == 0 { + return nil, ErrMissingPdbFiles + } + + return pdbs, nil +} + +// ParseDebugHeaderID TODO +func ParseDebugHeaderID(r io.ReadSeeker) (string, error) { + var magic uint32 + if err := binary.Read(r, binary.LittleEndian, &magic); err != nil { + return "", err + } + if magic != 0x424A5342 { + return "", ErrInvalidPdbMagicNumber + } + + if _, err := r.Seek(8, io.SeekCurrent); err != nil { + return "", err + } + + var versionStringSize int32 + if err := binary.Read(r, binary.LittleEndian, &versionStringSize); err != nil { + return "", err + } + if _, err := r.Seek(int64(versionStringSize), io.SeekCurrent); err != nil { + return "", err + } + if _, err := r.Seek(2, io.SeekCurrent); err != nil { + return "", err + } + + var streamCount int16 + if err := binary.Read(r, binary.LittleEndian, &streamCount); err != nil { + return "", err + } + + read4ByteAlignedString := func(r io.Reader) (string, error) { + b := make([]byte, 4) + var buf bytes.Buffer + for { + if _, err := r.Read(b); err != nil { + return "", err + } + if i := bytes.IndexByte(b, 0); i != -1 { + buf.Write(b[:i]) + return buf.String(), nil + } + buf.Write(b) + } + } + + for i := 0; i < int(streamCount); i++ { + var offset uint32 + if err := binary.Read(r, binary.LittleEndian, &offset); err != nil { + return "", err + } + if _, err := r.Seek(4, io.SeekCurrent); err != nil { + return "", err + } + name, err := read4ByteAlignedString(r) + if err != nil { + return "", err + } + + if name == "#Pdb" { + if _, err := r.Seek(int64(offset), io.SeekStart); err != nil { + return "", err + } + + b := make([]byte, 16) + if _, err := r.Read(b); err != nil { + return "", err + } + + data1 := binary.LittleEndian.Uint32(b[0:4]) + data2 := binary.LittleEndian.Uint16(b[4:6]) + data3 := binary.LittleEndian.Uint16(b[6:8]) + data4 := b[8:16] + + return fmt.Sprintf("%08x%04x%04x%04x%012x", data1, data2, data3, data4[:2], data4[2:]), nil + } + } + + return "", ErrMissingPdbStream +} diff --git a/modules/packages/nuget/symbol_extractor_test.go b/modules/packages/nuget/symbol_extractor_test.go new file mode 100644 index 0000000000..892d718caa --- /dev/null +++ b/modules/packages/nuget/symbol_extractor_test.go @@ -0,0 +1,82 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package nuget + +import ( + "archive/zip" + "bytes" + "encoding/base64" + "testing" + + "github.com/stretchr/testify/assert" +) + +const pdbContent = `QlNKQgEAAQAAAAAADAAAAFBEQiB2MS4wAAAAAAAABgB8AAAAWAAAACNQZGIAAAAA1AAAAAgBAAAj +fgAA3AEAAAQAAAAjU3RyaW5ncwAAAADgAQAABAAAACNVUwDkAQAAMAAAACNHVUlEAAAAFAIAACgB +AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==` + +func TestExtractPortablePdb(t *testing.T) { + createArchive := func(name string, content []byte) []byte { + var buf bytes.Buffer + archive := zip.NewWriter(&buf) + w, _ := archive.Create(name) + w.Write(content) + archive.Close() + return buf.Bytes() + } + + t.Run("MissingPdbFiles", func(t *testing.T) { + var buf bytes.Buffer + zip.NewWriter(&buf).Close() + + pdbs, err := ExtractPortablePdb(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + assert.ErrorIs(t, err, ErrMissingPdbFiles) + assert.Empty(t, pdbs) + }) + + t.Run("InvalidFiles", func(t *testing.T) { + data := createArchive("sub/test.bin", []byte{}) + + pdbs, err := ExtractPortablePdb(bytes.NewReader(data), int64(len(data))) + assert.ErrorIs(t, err, ErrInvalidFiles) + assert.Empty(t, pdbs) + }) + + t.Run("Valid", func(t *testing.T) { + b, _ := base64.StdEncoding.DecodeString(pdbContent) + data := createArchive("test.pdb", b) + + pdbs, err := ExtractPortablePdb(bytes.NewReader(data), int64(len(data))) + assert.NoError(t, err) + assert.Len(t, pdbs, 1) + assert.Equal(t, "test.pdb", pdbs[0].Name) + assert.Equal(t, "d910bb6948bd4c6cb40155bcf52c3c94", pdbs[0].ID) + pdbs.Close() + }) +} + +func TestParseDebugHeaderID(t *testing.T) { + t.Run("InvalidPdbMagicNumber", func(t *testing.T) { + id, err := ParseDebugHeaderID(bytes.NewReader([]byte{0, 0, 0, 0})) + assert.ErrorIs(t, err, ErrInvalidPdbMagicNumber) + assert.Empty(t, id) + }) + + t.Run("MissingPdbStream", func(t *testing.T) { + b, _ := base64.StdEncoding.DecodeString(`QlNKQgEAAQAAAAAADAAAAFBEQiB2MS4wAAAAAAAAAQB8AAAAWAAAACNVUwA=`) + + id, err := ParseDebugHeaderID(bytes.NewReader(b)) + assert.ErrorIs(t, err, ErrMissingPdbStream) + assert.Empty(t, id) + }) + + t.Run("Valid", func(t *testing.T) { + b, _ := base64.StdEncoding.DecodeString(pdbContent) + + id, err := ParseDebugHeaderID(bytes.NewReader(b)) + assert.NoError(t, err) + assert.Equal(t, "d910bb6948bd4c6cb40155bcf52c3c94", id) + }) +} diff --git a/modules/packages/pypi/metadata.go b/modules/packages/pypi/metadata.go new file mode 100644 index 0000000000..df367d10e2 --- /dev/null +++ b/modules/packages/pypi/metadata.go @@ -0,0 +1,16 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package pypi + +// Metadata represents the metadata of a PyPI package +type Metadata struct { + Author string `json:"author,omitempty"` + Description string `json:"description,omitempty"` + LongDescription string `json:"long_description,omitempty"` + Summary string `json:"summary,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + License string `json:"license,omitempty"` + RequiresPython string `json:"requires_python,omitempty"` +} diff --git a/modules/packages/rubygems/marshal.go b/modules/packages/rubygems/marshal.go new file mode 100644 index 0000000000..2c45042fa8 --- /dev/null +++ b/modules/packages/rubygems/marshal.go @@ -0,0 +1,311 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package rubygems + +import ( + "bufio" + "bytes" + "errors" + "io" + "reflect" +) + +const ( + majorVersion = 4 + minorVersion = 8 + + typeNil = '0' + typeTrue = 'T' + typeFalse = 'F' + typeFixnum = 'i' + typeString = '"' + typeSymbol = ':' + typeSymbolLink = ';' + typeArray = '[' + typeIVar = 'I' + typeUserMarshal = 'U' + typeUserDef = 'u' + typeObject = 'o' +) + +var ( + // ErrUnsupportedType indicates an unsupported type + ErrUnsupportedType = errors.New("Type is unsupported") + // ErrInvalidIntRange indicates an invalid number range + ErrInvalidIntRange = errors.New("Number is not in valid range") +) + +// RubyUserMarshal is a Ruby object that has a marshal_load function. +type RubyUserMarshal struct { + Name string + Value interface{} +} + +// RubyUserDef is a Ruby object that has a _load function. +type RubyUserDef struct { + Name string + Value interface{} +} + +// RubyObject is a default Ruby object. +type RubyObject struct { + Name string + Member map[string]interface{} +} + +// MarshalEncoder mimics Rubys Marshal class. +// Note: Only supports types used by the RubyGems package registry. +type MarshalEncoder struct { + w *bufio.Writer + symbols map[string]int +} + +// NewMarshalEncoder creates a new MarshalEncoder +func NewMarshalEncoder(w io.Writer) *MarshalEncoder { + return &MarshalEncoder{ + w: bufio.NewWriter(w), + symbols: map[string]int{}, + } +} + +// Encode encodes the given type +func (e *MarshalEncoder) Encode(v interface{}) error { + if _, err := e.w.Write([]byte{majorVersion, minorVersion}); err != nil { + return err + } + + if err := e.marshal(v); err != nil { + return err + } + + return e.w.Flush() +} + +func (e *MarshalEncoder) marshal(v interface{}) error { + if v == nil { + return e.marshalNil() + } + + val := reflect.ValueOf(v) + typ := reflect.TypeOf(v) + + if typ.Kind() == reflect.Ptr { + val = val.Elem() + typ = typ.Elem() + } + + switch typ.Kind() { + case reflect.Bool: + return e.marshalBool(val.Bool()) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32: + return e.marshalInt(val.Int()) + case reflect.String: + return e.marshalString(val.String()) + case reflect.Slice, reflect.Array: + return e.marshalArray(val) + } + + switch typ.Name() { + case "RubyUserMarshal": + return e.marshalUserMarshal(val.Interface().(RubyUserMarshal)) + case "RubyUserDef": + return e.marshalUserDef(val.Interface().(RubyUserDef)) + case "RubyObject": + return e.marshalObject(val.Interface().(RubyObject)) + } + + return ErrUnsupportedType +} + +func (e *MarshalEncoder) marshalNil() error { + return e.w.WriteByte(typeNil) +} + +func (e *MarshalEncoder) marshalBool(b bool) error { + if b { + return e.w.WriteByte(typeTrue) + } + return e.w.WriteByte(typeFalse) +} + +func (e *MarshalEncoder) marshalInt(i int64) error { + if err := e.w.WriteByte(typeFixnum); err != nil { + return err + } + + return e.marshalIntInternal(i) +} + +func (e *MarshalEncoder) marshalIntInternal(i int64) error { + if i == 0 { + return e.w.WriteByte(0) + } else if 0 < i && i < 123 { + return e.w.WriteByte(byte(i + 5)) + } else if -124 < i && i <= -1 { + return e.w.WriteByte(byte(i - 5)) + } + + var len int + if 122 < i && i <= 0xff { + len = 1 + } else if 0xff < i && i <= 0xffff { + len = 2 + } else if 0xffff < i && i <= 0xffffff { + len = 3 + } else if 0xffffff < i && i <= 0x3fffffff { + len = 4 + } else if -0x100 <= i && i < -123 { + len = -1 + } else if -0x10000 <= i && i < -0x100 { + len = -2 + } else if -0x1000000 <= i && i < -0x100000 { + len = -3 + } else if -0x40000000 <= i && i < -0x1000000 { + len = -4 + } else { + return ErrInvalidIntRange + } + + if err := e.w.WriteByte(byte(len)); err != nil { + return err + } + if len < 0 { + len = -len + } + + for c := 0; c < len; c++ { + if err := e.w.WriteByte(byte(i >> uint(8*c) & 0xff)); err != nil { + return err + } + } + + return nil +} + +func (e *MarshalEncoder) marshalString(str string) error { + if err := e.w.WriteByte(typeIVar); err != nil { + return err + } + + if err := e.marshalRawString(str); err != nil { + return err + } + + if err := e.marshalIntInternal(1); err != nil { + return err + } + + if err := e.marshalSymbol("E"); err != nil { + return err + } + + return e.marshalBool(true) +} + +func (e *MarshalEncoder) marshalRawString(str string) error { + if err := e.w.WriteByte(typeString); err != nil { + return err + } + + if err := e.marshalIntInternal(int64(len(str))); err != nil { + return err + } + + _, err := e.w.WriteString(str) + return err +} + +func (e *MarshalEncoder) marshalSymbol(str string) error { + if index, ok := e.symbols[str]; ok { + if err := e.w.WriteByte(typeSymbolLink); err != nil { + return err + } + return e.marshalIntInternal(int64(index)) + } + + e.symbols[str] = len(e.symbols) + + if err := e.w.WriteByte(typeSymbol); err != nil { + return err + } + + if err := e.marshalIntInternal(int64(len(str))); err != nil { + return err + } + + _, err := e.w.WriteString(str) + return err +} + +func (e *MarshalEncoder) marshalArray(arr reflect.Value) error { + if err := e.w.WriteByte(typeArray); err != nil { + return err + } + + len := arr.Len() + + if err := e.marshalIntInternal(int64(len)); err != nil { + return err + } + + for i := 0; i < len; i++ { + if err := e.marshal(arr.Index(i).Interface()); err != nil { + return err + } + } + return nil +} + +func (e *MarshalEncoder) marshalUserMarshal(userMarshal RubyUserMarshal) error { + if err := e.w.WriteByte(typeUserMarshal); err != nil { + return err + } + + if err := e.marshalSymbol(userMarshal.Name); err != nil { + return err + } + + return e.marshal(userMarshal.Value) +} + +func (e *MarshalEncoder) marshalUserDef(userDef RubyUserDef) error { + var buf bytes.Buffer + if err := NewMarshalEncoder(&buf).Encode(userDef.Value); err != nil { + return err + } + + if err := e.w.WriteByte(typeUserDef); err != nil { + return err + } + if err := e.marshalSymbol(userDef.Name); err != nil { + return err + } + if err := e.marshalIntInternal(int64(buf.Len())); err != nil { + return err + } + _, err := e.w.Write(buf.Bytes()) + return err +} + +func (e *MarshalEncoder) marshalObject(obj RubyObject) error { + if err := e.w.WriteByte(typeObject); err != nil { + return err + } + if err := e.marshalSymbol(obj.Name); err != nil { + return err + } + if err := e.marshalIntInternal(int64(len(obj.Member))); err != nil { + return err + } + for k, v := range obj.Member { + if err := e.marshalSymbol(k); err != nil { + return err + } + if err := e.marshal(v); err != nil { + return err + } + } + return nil +} diff --git a/modules/packages/rubygems/marshal_test.go b/modules/packages/rubygems/marshal_test.go new file mode 100644 index 0000000000..e5963ebcd6 --- /dev/null +++ b/modules/packages/rubygems/marshal_test.go @@ -0,0 +1,99 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package rubygems + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMinimalEncoder(t *testing.T) { + cases := []struct { + Value interface{} + Expected []byte + Error error + }{ + { + Value: nil, + Expected: []byte{4, 8, 0x30}, + }, + { + Value: true, + Expected: []byte{4, 8, 'T'}, + }, + { + Value: false, + Expected: []byte{4, 8, 'F'}, + }, + { + Value: 0, + Expected: []byte{4, 8, 'i', 0}, + }, + { + Value: 1, + Expected: []byte{4, 8, 'i', 6}, + }, + { + Value: -1, + Expected: []byte{4, 8, 'i', 0xfa}, + }, + { + Value: 0x1fffffff, + Expected: []byte{4, 8, 'i', 4, 0xff, 0xff, 0xff, 0x1f}, + }, + { + Value: 0x41000000, + Error: ErrInvalidIntRange, + }, + { + Value: "test", + Expected: []byte{4, 8, 'I', '"', 9, 't', 'e', 's', 't', 6, ':', 6, 'E', 'T'}, + }, + { + Value: []int{1, 2}, + Expected: []byte{4, 8, '[', 7, 'i', 6, 'i', 7}, + }, + { + Value: &RubyUserMarshal{ + Name: "Test", + Value: 4, + }, + Expected: []byte{4, 8, 'U', ':', 9, 'T', 'e', 's', 't', 'i', 9}, + }, + { + Value: &RubyUserDef{ + Name: "Test", + Value: 4, + }, + Expected: []byte{4, 8, 'u', ':', 9, 'T', 'e', 's', 't', 9, 4, 8, 'i', 9}, + }, + { + Value: &RubyObject{ + Name: "Test", + Member: map[string]interface{}{ + "test": 4, + }, + }, + Expected: []byte{4, 8, 'o', ':', 9, 'T', 'e', 's', 't', 6, ':', 9, 't', 'e', 's', 't', 'i', 9}, + }, + { + Value: &struct { + Name string + }{ + "test", + }, + Error: ErrUnsupportedType, + }, + } + + for i, c := range cases { + var b bytes.Buffer + err := NewMarshalEncoder(&b).Encode(c.Value) + assert.ErrorIs(t, err, c.Error) + assert.Equal(t, c.Expected, b.Bytes(), "case %d", i) + } +} diff --git a/modules/packages/rubygems/metadata.go b/modules/packages/rubygems/metadata.go new file mode 100644 index 0000000000..942f205fc3 --- /dev/null +++ b/modules/packages/rubygems/metadata.go @@ -0,0 +1,222 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package rubygems + +import ( + "archive/tar" + "compress/gzip" + "errors" + "io" + "regexp" + "strings" + + "code.gitea.io/gitea/modules/validation" + + "gopkg.in/yaml.v2" +) + +var ( + // ErrMissingMetadataFile indicates a missing metadata.gz file + ErrMissingMetadataFile = errors.New("Metadata file is missing") + // ErrInvalidName indicates an invalid id in the metadata.gz file + ErrInvalidName = errors.New("Metadata file contains an invalid name") + // ErrInvalidVersion indicates an invalid version in the metadata.gz file + ErrInvalidVersion = errors.New("Metadata file contains an invalid version") +) + +var versionMatcher = regexp.MustCompile(`\A[0-9]+(?:\.[0-9a-zA-Z]+)*(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?\z`) + +// Package represents a RubyGems package +type Package struct { + Name string + Version string + Metadata *Metadata +} + +// Metadata represents the metadata of a RubyGems package +type Metadata struct { + Platform string `json:"platform,omitempty"` + Description string `json:"description,omitempty"` + Summary string `json:"summary,omitempty"` + Authors []string `json:"authors,omitempty"` + Licenses []string `json:"licenses,omitempty"` + RequiredRubyVersion []VersionRequirement `json:"required_ruby_version,omitempty"` + RequiredRubygemsVersion []VersionRequirement `json:"required_rubygems_version,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + RuntimeDependencies []Dependency `json:"runtime_dependencies,omitempty"` + DevelopmentDependencies []Dependency `json:"development_dependencies,omitempty"` +} + +// VersionRequirement represents a version restriction +type VersionRequirement struct { + Restriction string `json:"restriction"` + Version string `json:"version"` +} + +// Dependency represents a dependency of a RubyGems package +type Dependency struct { + Name string `json:"name"` + Version []VersionRequirement `json:"version"` +} + +type gemspec struct { + Name string `yaml:"name"` + Version struct { + Version string `yaml:"version"` + } `yaml:"version"` + Platform string `yaml:"platform"` + Authors []string `yaml:"authors"` + Autorequire interface{} `yaml:"autorequire"` + Bindir string `yaml:"bindir"` + CertChain []interface{} `yaml:"cert_chain"` + Date string `yaml:"date"` + Dependencies []struct { + Name string `yaml:"name"` + Requirement requirement `yaml:"requirement"` + Type string `yaml:"type"` + Prerelease bool `yaml:"prerelease"` + VersionRequirements requirement `yaml:"version_requirements"` + } `yaml:"dependencies"` + Description string `yaml:"description"` + Email string `yaml:"email"` + Executables []string `yaml:"executables"` + Extensions []interface{} `yaml:"extensions"` + ExtraRdocFiles []string `yaml:"extra_rdoc_files"` + Files []string `yaml:"files"` + Homepage string `yaml:"homepage"` + Licenses []string `yaml:"licenses"` + Metadata struct { + BugTrackerURI string `yaml:"bug_tracker_uri"` + ChangelogURI string `yaml:"changelog_uri"` + DocumentationURI string `yaml:"documentation_uri"` + SourceCodeURI string `yaml:"source_code_uri"` + } `yaml:"metadata"` + PostInstallMessage interface{} `yaml:"post_install_message"` + RdocOptions []interface{} `yaml:"rdoc_options"` + RequirePaths []string `yaml:"require_paths"` + RequiredRubyVersion requirement `yaml:"required_ruby_version"` + RequiredRubygemsVersion requirement `yaml:"required_rubygems_version"` + Requirements []interface{} `yaml:"requirements"` + RubygemsVersion string `yaml:"rubygems_version"` + SigningKey interface{} `yaml:"signing_key"` + SpecificationVersion int `yaml:"specification_version"` + Summary string `yaml:"summary"` + TestFiles []interface{} `yaml:"test_files"` +} + +type requirement struct { + Requirements [][]interface{} `yaml:"requirements"` +} + +// AsVersionRequirement converts into []VersionRequirement +func (r requirement) AsVersionRequirement() []VersionRequirement { + requirements := make([]VersionRequirement, 0, len(r.Requirements)) + for _, req := range r.Requirements { + if len(req) != 2 { + continue + } + restriction, ok := req[0].(string) + if !ok { + continue + } + vm, ok := req[1].(map[interface{}]interface{}) + if !ok { + continue + } + versionInt, ok := vm["version"] + if !ok { + continue + } + version, ok := versionInt.(string) + if !ok || version == "0" { + continue + } + + requirements = append(requirements, VersionRequirement{ + Restriction: restriction, + Version: version, + }) + } + return requirements +} + +// ParsePackageMetaData parses the metadata of a Gem package file +func ParsePackageMetaData(r io.Reader) (*Package, error) { + archive := tar.NewReader(r) + for { + hdr, err := archive.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if hdr.Name == "metadata.gz" { + return parseMetadataFile(archive) + } + } + + return nil, ErrMissingMetadataFile +} + +func parseMetadataFile(r io.Reader) (*Package, error) { + zr, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + defer zr.Close() + + var spec gemspec + if err := yaml.NewDecoder(zr).Decode(&spec); err != nil { + return nil, err + } + + if len(spec.Name) == 0 || strings.Contains(spec.Name, "/") { + return nil, ErrInvalidName + } + + if !versionMatcher.MatchString(spec.Version.Version) { + return nil, ErrInvalidVersion + } + + if !validation.IsValidURL(spec.Homepage) { + spec.Homepage = "" + } + if !validation.IsValidURL(spec.Metadata.SourceCodeURI) { + spec.Metadata.SourceCodeURI = "" + } + + m := &Metadata{ + Platform: spec.Platform, + Description: spec.Description, + Summary: spec.Summary, + Authors: spec.Authors, + Licenses: spec.Licenses, + ProjectURL: spec.Homepage, + RequiredRubyVersion: spec.RequiredRubyVersion.AsVersionRequirement(), + RequiredRubygemsVersion: spec.RequiredRubygemsVersion.AsVersionRequirement(), + DevelopmentDependencies: make([]Dependency, 0, 5), + RuntimeDependencies: make([]Dependency, 0, 5), + } + + for _, gemdep := range spec.Dependencies { + dep := Dependency{ + Name: gemdep.Name, + Version: gemdep.Requirement.AsVersionRequirement(), + } + if gemdep.Type == ":runtime" { + m.RuntimeDependencies = append(m.RuntimeDependencies, dep) + } else { + m.DevelopmentDependencies = append(m.DevelopmentDependencies, dep) + } + } + + return &Package{ + Name: spec.Name, + Version: spec.Version.Version, + Metadata: m, + }, nil +} diff --git a/modules/packages/rubygems/metadata_test.go b/modules/packages/rubygems/metadata_test.go new file mode 100644 index 0000000000..dbefa9c236 --- /dev/null +++ b/modules/packages/rubygems/metadata_test.go @@ -0,0 +1,89 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package rubygems + +import ( + "archive/tar" + "bytes" + "encoding/base64" + "io" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParsePackageMetaData(t *testing.T) { + createArchive := func(filename string, content []byte) io.Reader { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + hdr := &tar.Header{ + Name: filename, + Mode: 0o600, + Size: int64(len(content)), + } + tw.WriteHeader(hdr) + tw.Write(content) + tw.Close() + return &buf + } + + t.Run("MissingMetadataFile", func(t *testing.T) { + data := createArchive("dummy.txt", []byte{0}) + + rp, err := ParsePackageMetaData(data) + assert.ErrorIs(t, err, ErrMissingMetadataFile) + assert.Nil(t, rp) + }) + + t.Run("Valid", func(t *testing.T) { + content, _ := base64.StdEncoding.DecodeString("H4sICHC/I2EEAG1ldGFkYXRhAAEeAOH/bmFtZTogZwp2ZXJzaW9uOgogIHZlcnNpb246IDEKWw35Tx4AAAA=") + data := createArchive("metadata.gz", content) + + rp, err := ParsePackageMetaData(data) + assert.NoError(t, err) + assert.NotNil(t, rp) + }) +} + +func TestParseMetadataFile(t *testing.T) { + content, _ := base64.StdEncoding.DecodeString(`H4sIAMe7I2ECA9VVTW/UMBC9+1eYXvaUbJpSQBZUHJAqDlwK4kCFIseZzZrGH9iTqisEv52Js9nd +0KqggiqRXWnX45n3ZuZ5nCzL+JPQ15ulq7+AQnEORoj3HpReaSVRO8usNCB4qxEku4YQySbuCPo4 +bjHOd07HeZGfMt9JXLlgBB9imOxx7UIULOPnCZMMLsDXXgeiYbW2jQ6C0y9TELBSa6kJ6/IzaySS +R1mUx1nxIitPeFGI9M2L6eGfWAMebANWaUgktzN9M3lsKNmxutBb1AYyCibbNhsDFu+q9GK/Tc4z +d2IcLBl9js5eHaXFsLyvXeNz0LQyL/YoLx8EsiCMBZlx46k6sS2PDD5AgA5kJPNKdhH2elWzOv7n +uv9Q9Aau/6ngP84elvNpXh5oRVlB5/yW7BH0+qu0G4gqaI/JdEHBFBS5l+pKtsARIjIwUnfj8Le0 ++TrdJLl2DG5A9SjrjgZ1mG+4QbAD+G4ZZBUap6qVnnzGf6Rwp+vliBRqtnYGPBEKvkb0USyXE8mS +dVoR6hj07u0HZgAl3SRS8G/fmXcRK20jyq6rDMSYQFgidamqkXbbuspLXE/0k7GphtKqe67GuRC/ +yjAbmt9LsOMp8xMamFkSQ38fP5EFjdz8LA4do2C69VvqWXAJgrPbKZb58/xZXrKoW6ttW13Bhvzi +4ftn7/yUxd4YGcglvTmmY8aGY3ZwRn4CqcWcidUGAAA=`) + rp, err := parseMetadataFile(bytes.NewReader(content)) + assert.NoError(t, err) + assert.NotNil(t, rp) + + assert.Equal(t, "gitea", rp.Name) + assert.Equal(t, "1.0.5", rp.Version) + assert.Equal(t, "ruby", rp.Metadata.Platform) + assert.Equal(t, "Gitea package", rp.Metadata.Summary) + assert.Equal(t, "RubyGems package test", rp.Metadata.Description) + assert.Equal(t, []string{"Gitea"}, rp.Metadata.Authors) + assert.Equal(t, "https://gitea.io/", rp.Metadata.ProjectURL) + assert.Equal(t, []string{"MIT"}, rp.Metadata.Licenses) + assert.Empty(t, rp.Metadata.RequiredRubygemsVersion) + assert.Len(t, rp.Metadata.RequiredRubyVersion, 1) + assert.Equal(t, ">=", rp.Metadata.RequiredRubyVersion[0].Restriction) + assert.Equal(t, "2.3.0", rp.Metadata.RequiredRubyVersion[0].Version) + assert.Len(t, rp.Metadata.RuntimeDependencies, 1) + assert.Equal(t, "runtime-dep", rp.Metadata.RuntimeDependencies[0].Name) + assert.Len(t, rp.Metadata.RuntimeDependencies[0].Version, 2) + assert.Equal(t, ">=", rp.Metadata.RuntimeDependencies[0].Version[0].Restriction) + assert.Equal(t, "1.2.0", rp.Metadata.RuntimeDependencies[0].Version[0].Version) + assert.Equal(t, "<", rp.Metadata.RuntimeDependencies[0].Version[1].Restriction) + assert.Equal(t, "2.0", rp.Metadata.RuntimeDependencies[0].Version[1].Version) + assert.Len(t, rp.Metadata.DevelopmentDependencies, 1) + assert.Equal(t, "dev-dep", rp.Metadata.DevelopmentDependencies[0].Name) + assert.Len(t, rp.Metadata.DevelopmentDependencies[0].Version, 1) + assert.Equal(t, "~>", rp.Metadata.DevelopmentDependencies[0].Version[0].Restriction) + assert.Equal(t, "5.2", rp.Metadata.DevelopmentDependencies[0].Version[0].Version) +} diff --git a/modules/setting/packages.go b/modules/setting/packages.go new file mode 100644 index 0000000000..65653b990e --- /dev/null +++ b/modules/setting/packages.go @@ -0,0 +1,47 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package setting + +import ( + "os" + "path/filepath" + + "code.gitea.io/gitea/modules/log" +) + +// Package registry settings +var ( + Packages = struct { + Storage + Enabled bool + ChunkedUploadPath string + RegistryHost string + }{ + Enabled: true, + } +) + +func newPackages() { + sec := Cfg.Section("packages") + if err := sec.MapTo(&Packages); err != nil { + log.Fatal("Failed to map Packages settings: %v", err) + } + + Packages.Storage = getStorage("packages", "", nil) + + Packages.RegistryHost = Domain + if (Protocol == HTTP && HTTPPort != "80") || (Protocol == HTTPS && HTTPPort != "443") { + Packages.RegistryHost += ":" + HTTPPort + } + + Packages.ChunkedUploadPath = filepath.ToSlash(sec.Key("CHUNKED_UPLOAD_PATH").MustString("tmp/package-upload")) + if !filepath.IsAbs(Packages.ChunkedUploadPath) { + Packages.ChunkedUploadPath = filepath.ToSlash(filepath.Join(AppDataPath, Packages.ChunkedUploadPath)) + } + + if err := os.MkdirAll(Packages.ChunkedUploadPath, os.ModePerm); err != nil { + log.Error("Unable to create chunked upload directory: %s (%v)", Packages.ChunkedUploadPath, err) + } +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index c80fc3d204..17a02bf5a1 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -212,6 +212,7 @@ var ( MembersPagingNum int FeedMaxCommitNum int FeedPagingNum int + PackagesPagingNum int GraphMaxCommitNum int CodeCommentLines int ReactionMaxUserNum int @@ -264,6 +265,7 @@ var ( MembersPagingNum: 20, FeedMaxCommitNum: 5, FeedPagingNum: 20, + PackagesPagingNum: 20, GraphMaxCommitNum: 100, CodeCommentLines: 4, ReactionMaxUserNum: 10, @@ -1016,6 +1018,8 @@ func loadFromConf(allowEmpty bool, extraConfig string) { newPictureService() + newPackages() + if err = Cfg.Section("ui").MapTo(&UI); err != nil { log.Fatal("Failed to map UI settings: %v", err) } else if err = Cfg.Section("markdown").MapTo(&Markdown); err != nil { diff --git a/modules/storage/storage.go b/modules/storage/storage.go index f11e1ac743..ef7f6029a5 100644 --- a/modules/storage/storage.go +++ b/modules/storage/storage.go @@ -123,6 +123,9 @@ var ( // RepoArchives represents repository archives storage RepoArchives ObjectStorage + + // Packages represents packages storage + Packages ObjectStorage ) // Init init the stoarge @@ -143,7 +146,11 @@ func Init() error { return err } - return initRepoArchives() + if err := initRepoArchives(); err != nil { + return err + } + + return initPackages() } // NewStorage takes a storage type and some config and returns an ObjectStorage or an error @@ -188,3 +195,9 @@ func initRepoArchives() (err error) { RepoArchives, err = NewStorage(setting.RepoArchive.Storage.Type, &setting.RepoArchive.Storage) return } + +func initPackages() (err error) { + log.Info("Initialising Packages storage with type: %s", setting.Packages.Storage.Type) + Packages, err = NewStorage(setting.Packages.Storage.Type, &setting.Packages.Storage) + return +} diff --git a/modules/structs/hook.go b/modules/structs/hook.go index e4d7652c72..07d51915de 100644 --- a/modules/structs/hook.go +++ b/modules/structs/hook.go @@ -110,6 +110,7 @@ var ( _ Payloader = &PullRequestPayload{} _ Payloader = &RepositoryPayload{} _ Payloader = &ReleasePayload{} + _ Payloader = &PackagePayload{} ) // _________ __ @@ -425,3 +426,27 @@ type RepositoryPayload struct { func (p *RepositoryPayload) JSONPayload() ([]byte, error) { return json.MarshalIndent(p, "", " ") } + +// HookPackageAction an action that happens to a package +type HookPackageAction string + +const ( + // HookPackageCreated created + HookPackageCreated HookPackageAction = "created" + // HookPackageDeleted deleted + HookPackageDeleted HookPackageAction = "deleted" +) + +// PackagePayload represents a package payload +type PackagePayload struct { + Action HookPackageAction `json:"action"` + Repository *Repository `json:"repository"` + Package *Package `json:"package"` + Organization *User `json:"organization"` + Sender *User `json:"sender"` +} + +// JSONPayload implements Payload +func (p *PackagePayload) JSONPayload() ([]byte, error) { + return json.MarshalIndent(p, "", " ") +} diff --git a/modules/structs/package.go b/modules/structs/package.go new file mode 100644 index 0000000000..fbdd6c90aa --- /dev/null +++ b/modules/structs/package.go @@ -0,0 +1,33 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package structs + +import ( + "time" +) + +// Package represents a package +type Package struct { + ID int64 `json:"id"` + Owner *User `json:"owner"` + Repository *Repository `json:"repository"` + Creator *User `json:"creator"` + Type string `json:"type"` + Name string `json:"name"` + Version string `json:"version"` + // swagger:strfmt date-time + CreatedAt time.Time `json:"created_at"` +} + +// PackageFile represents a package file +type PackageFile struct { + ID int64 `json:"id"` + Size int64 + Name string `json:"name"` + HashMD5 string `json:"md5"` + HashSHA1 string `json:"sha1"` + HashSHA256 string `json:"sha256"` + HashSHA512 string `json:"sha512"` +} diff --git a/modules/templates/helper.go b/modules/templates/helper.go index 80ad7066a7..1201710b92 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -34,6 +34,7 @@ import ( "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/svg" @@ -161,7 +162,16 @@ func NewFuncMap() []template.FuncMap { "RenderEmojiPlain": emoji.ReplaceAliases, "ReactionToEmoji": ReactionToEmoji, "RenderNote": RenderNote, - "IsMultilineCommitMessage": IsMultilineCommitMessage, + "RenderMarkdownToHtml": func(input string) template.HTML { + output, err := markdown.RenderString(&markup.RenderContext{ + URLPrefix: setting.AppSubURL, + }, input) + if err != nil { + log.Error("RenderString: %v", err) + } + return template.HTML(output) + }, + "IsMultilineCommitMessage": IsMultilineCommitMessage, "ThemeColorMetaTag": func() string { return setting.UI.ThemeColorMetaTag }, diff --git a/modules/util/filebuffer/file_backed_buffer.go b/modules/util/filebuffer/file_backed_buffer.go new file mode 100644 index 0000000000..128030b4c5 --- /dev/null +++ b/modules/util/filebuffer/file_backed_buffer.go @@ -0,0 +1,147 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package filebuffer + +import ( + "bytes" + "errors" + "io" + "os" +) + +const maxInt = int(^uint(0) >> 1) // taken from bytes.Buffer + +var ( + // ErrInvalidMemorySize occurs if the memory size is not in a valid range + ErrInvalidMemorySize = errors.New("Memory size must be greater 0 and lower math.MaxInt32") + // ErrWriteAfterRead occurs if Write is called after a read operation + ErrWriteAfterRead = errors.New("Write is unsupported after a read operation") +) + +type readAtSeeker interface { + io.ReadSeeker + io.ReaderAt +} + +// FileBackedBuffer uses a memory buffer with a fixed size. +// If more data is written a temporary file is used instead. +// It implements io.ReadWriteCloser, io.ReadSeekCloser and io.ReaderAt +type FileBackedBuffer struct { + maxMemorySize int64 + size int64 + buffer bytes.Buffer + file *os.File + reader readAtSeeker +} + +// New creates a file backed buffer with a specific maximum memory size +func New(maxMemorySize int) (*FileBackedBuffer, error) { + if maxMemorySize < 0 || maxMemorySize > maxInt { + return nil, ErrInvalidMemorySize + } + + return &FileBackedBuffer{ + maxMemorySize: int64(maxMemorySize), + }, nil +} + +// CreateFromReader creates a file backed buffer and copies the provided reader data into it. +func CreateFromReader(r io.Reader, maxMemorySize int) (*FileBackedBuffer, error) { + b, err := New(maxMemorySize) + if err != nil { + return nil, err + } + + _, err = io.Copy(b, r) + if err != nil { + return nil, err + } + + return b, nil +} + +// Write implements io.Writer +func (b *FileBackedBuffer) Write(p []byte) (int, error) { + if b.reader != nil { + return 0, ErrWriteAfterRead + } + + var n int + var err error + + if b.file != nil { + n, err = b.file.Write(p) + } else { + if b.size+int64(len(p)) > b.maxMemorySize { + b.file, err = os.CreateTemp("", "gitea-buffer-") + if err != nil { + return 0, err + } + + _, err = io.Copy(b.file, &b.buffer) + if err != nil { + return 0, err + } + + return b.Write(p) + } + + n, err = b.buffer.Write(p) + } + + if err != nil { + return n, err + } + b.size += int64(n) + return n, nil +} + +// Size returns the byte size of the buffered data +func (b *FileBackedBuffer) Size() int64 { + return b.size +} + +func (b *FileBackedBuffer) switchToReader() { + if b.reader != nil { + return + } + + if b.file != nil { + b.reader = b.file + } else { + b.reader = bytes.NewReader(b.buffer.Bytes()) + } +} + +// Read implements io.Reader +func (b *FileBackedBuffer) Read(p []byte) (int, error) { + b.switchToReader() + + return b.reader.Read(p) +} + +// ReadAt implements io.ReaderAt +func (b *FileBackedBuffer) ReadAt(p []byte, off int64) (int, error) { + b.switchToReader() + + return b.reader.ReadAt(p, off) +} + +// Seek implements io.Seeker +func (b *FileBackedBuffer) Seek(offset int64, whence int) (int64, error) { + b.switchToReader() + + return b.reader.Seek(offset, whence) +} + +// Close implements io.Closer +func (b *FileBackedBuffer) Close() error { + if b.file != nil { + err := b.file.Close() + os.Remove(b.file.Name()) + return err + } + return nil +} |