Add Package Registry (#16510)

* Added package store settings.

* Added models.

* Added generic package registry.

* Added tests.

* Added NuGet package registry.

* Moved service index to api file.

* Added NPM package registry.

* Added Maven package registry.

* Added PyPI package registry.

* Summary is deprecated.

* Changed npm name.

* Sanitize project url.

* Allow only scoped packages.

* Added user interface.

* Changed method name.

* Added missing migration file.

* Set page info.

* Added documentation.

* Added documentation links.

* Fixed wrong error message.

* Lint template files.

* Fixed merge errors.

* Fixed unit test storage path.

* Switch to json module.

* Added suggestions.

* Added package webhook.

* Add package api.

* Fixed swagger file.

* Fixed enum and comments.

* Fixed NuGet pagination.

* Print test names.

* Added api tests.

* Fixed access level.

* Fix User unmarshal.

* Added RubyGems package registry.

* Fix lint.

* Implemented io.Writer.

* Added support for sha256/sha512 checksum files.

* Improved maven-metadata.xml support.

* Added support for symbol package uploads.

* Added tests.

* Added overview docs.

* Added npm dependencies and keywords.

* Added no-packages information.

* Display file size.

* Display asset count.

* Fixed filter alignment.

* Added package icons.

* Formatted instructions.

* Allow anonymous package downloads.

* Fixed comments.

* Fixed postgres test.

* Moved file.

* Moved models to models/packages.

* Use correct error response format per client.

* Use simpler search form.

* Fixed IsProd.

* Restructured data model.

* Prevent empty filename.

* Fix swagger.

* Implemented user/org registry.

* Implemented UI.

* Use GetUserByIDCtx.

* Use table for dependencies.

* make svg

* Added support for unscoped npm packages.

* Add support for npm dist tags.

* Added tests for npm tags.

* Unlink packages if repository gets deleted.

* Prevent user/org delete if a packages exist.

* Use package unlink in repository service.

* Added support for composer packages.

* Restructured package docs.

* Added missing tests.

* Fixed generic content page.

* Fixed docs.

* Fixed swagger.

* Added missing type.

* Fixed ambiguous column.

* Organize content store by sha256 hash.

* Added admin package management.

* Added support for sorting.

* Add support for multiple identical versions/files.

* Added missing repository unlink.

* Added file properties.

* make fmt

* lint

* Added Conan package registry.

* Updated docs.

* Unify package names.

* Added swagger enum.

* Use longer TEXT column type.

* Removed version composite key.

* Merged package and container registry.

* Removed index.

* Use dedicated package router.

* Moved files to new location.

* Updated docs.

* Fixed JOIN order.

* Fixed GROUP BY statement.

* Fixed GROUP BY #2.

* Added symbol server support.

* Added more tests.

* Set NOT NULL.

* Added setting to disable package registries.

* Moved auth into service.

* refactor

* Use ctx everywhere.

* Added package cleanup task.

* Changed packages path.

* Added container registry.

* Refactoring

* Updated comparison.

* Fix swagger.

* Fixed table order.

* Use token auth for npm routes.

* Enabled ReverseProxy auth.

* Added packages link for orgs.

* Fixed anonymous org access.

* Enable copy button for setup instructions.

* Merge error

* Added suggestions.

* Fixed merge.

* Handle "generic".

* Added link for TODO.

* Added suggestions.

* Changed temporary buffer filename.

* Added suggestions.

* Apply suggestions from code review

Co-authored-by: Thomas Boerger <thomas@webhippie.de>

* Update docs/content/doc/packages/nuget.en-us.md

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: Thomas Boerger <thomas@webhippie.de>

4 files changed, 735 insertions, 0 deletions

diff --git a/services/packages/auth.go b/services/packages/auth.go
new file mode 100644
index 0000000000..50212fccfd
--- /dev/null
+++ b/services/packages/auth.go
@@ -0,0 +1,66 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package packages
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/golang-jwt/jwt/v4"
+)
+
+type packageClaims struct {
+	jwt.RegisteredClaims
+	UserID int64
+}
+
+func CreateAuthorizationToken(u *user_model.User) (string, error) {
+	now := time.Now()
+
+	claims := packageClaims{
+		RegisteredClaims: jwt.RegisteredClaims{
+			ExpiresAt: jwt.NewNumericDate(now.Add(24 * time.Hour)),
+			NotBefore: jwt.NewNumericDate(now),
+		},
+		UserID: u.ID,
+	}
+	token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
+
+	tokenString, err := token.SignedString([]byte(setting.SecretKey))
+	if err != nil {
+		return "", err
+	}
+
+	return tokenString, nil
+}
+
+func ParseAuthorizationToken(req *http.Request) (int64, error) {
+	parts := strings.SplitN(req.Header.Get("Authorization"), " ", 2)
+	if len(parts) != 2 {
+		return 0, fmt.Errorf("no token")
+	}
+
+	token, err := jwt.ParseWithClaims(parts[1], &packageClaims{}, func(t *jwt.Token) (interface{}, error) {
+		if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok {
+			return nil, fmt.Errorf("unexpected signing method: %v", t.Header["alg"])
+		}
+		return []byte(setting.SecretKey), nil
+	})
+	if err != nil {
+		return 0, err
+	}
+
+	c, ok := token.Claims.(*packageClaims)
+	if !token.Valid || !ok {
+		return 0, fmt.Errorf("invalid token claim")
+	}
+
+	return c.UserID, nil
+}
diff --git a/services/packages/container/blob_uploader.go b/services/packages/container/blob_uploader.go
new file mode 100644
index 0000000000..762f9e5259
--- /dev/null
+++ b/services/packages/container/blob_uploader.go
@@ -0,0 +1,136 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package container
+
+import (
+	"context"
+	"errors"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	packages_model "code.gitea.io/gitea/models/packages"
+	packages_module "code.gitea.io/gitea/modules/packages"
+	"code.gitea.io/gitea/modules/setting"
+)
+
+var (
+	// errWriteAfterRead occurs if Write is called after a read operation
+	errWriteAfterRead = errors.New("write is unsupported after a read operation")
+	// errOffsetMissmatch occurs if the file offset is different than the model
+	errOffsetMissmatch = errors.New("offset mismatch between file and model")
+)
+
+// BlobUploader handles chunked blob uploads
+type BlobUploader struct {
+	*packages_model.PackageBlobUpload
+	*packages_module.MultiHasher
+	file    *os.File
+	reading bool
+}
+
+func buildFilePath(id string) string {
+	return filepath.Join(setting.Packages.ChunkedUploadPath, path.Clean("/" + strings.ReplaceAll(id, "\\", "/"))[1:])
+}
+
+// NewBlobUploader creates a new blob uploader for the given id
+func NewBlobUploader(ctx context.Context, id string) (*BlobUploader, error) {
+	model, err := packages_model.GetBlobUploadByID(ctx, id)
+	if err != nil {
+		return nil, err
+	}
+
+	hash := packages_module.NewMultiHasher()
+	if len(model.HashStateBytes) != 0 {
+		if err := hash.UnmarshalBinary(model.HashStateBytes); err != nil {
+			return nil, err
+		}
+	}
+
+	f, err := os.OpenFile(buildFilePath(model.ID), os.O_RDWR|os.O_CREATE, 0o666)
+	if err != nil {
+		return nil, err
+	}
+
+	return &BlobUploader{
+		model,
+		hash,
+		f,
+		false,
+	}, nil
+}
+
+// Close implements io.Closer
+func (u *BlobUploader) Close() error {
+	return u.file.Close()
+}
+
+// Append appends a chunk of data and updates the model
+func (u *BlobUploader) Append(ctx context.Context, r io.Reader) error {
+	if u.reading {
+		return errWriteAfterRead
+	}
+
+	offset, err := u.file.Seek(0, io.SeekEnd)
+	if err != nil {
+		return err
+	}
+	if offset != u.BytesReceived {
+		return errOffsetMissmatch
+	}
+
+	n, err := io.Copy(io.MultiWriter(u.file, u.MultiHasher), r)
+	if err != nil {
+		return err
+	}
+
+	// fast path if nothing was written
+	if n == 0 {
+		return nil
+	}
+
+	u.BytesReceived += n
+
+	u.HashStateBytes, err = u.MultiHasher.MarshalBinary()
+	if err != nil {
+		return err
+	}
+
+	return packages_model.UpdateBlobUpload(ctx, u.PackageBlobUpload)
+}
+
+func (u *BlobUploader) Size() int64 {
+	return u.BytesReceived
+}
+
+// Read implements io.Reader
+func (u *BlobUploader) Read(p []byte) (int, error) {
+	if !u.reading {
+		_, err := u.file.Seek(0, io.SeekStart)
+		if err != nil {
+			return 0, err
+		}
+
+		u.reading = true
+	}
+
+	return u.file.Read(p)
+}
+
+// Remove deletes the data and the model of a blob upload
+func RemoveBlobUploadByID(ctx context.Context, id string) error {
+	if err := packages_model.DeleteBlobUploadByID(ctx, id); err != nil {
+		return err
+	}
+
+	err := os.Remove(buildFilePath(id))
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+
+	return nil
+}
diff --git a/services/packages/container/cleanup.go b/services/packages/container/cleanup.go
new file mode 100644
index 0000000000..91992a4d7f
--- /dev/null
+++ b/services/packages/container/cleanup.go
@@ -0,0 +1,75 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package container
+
+import (
+	"context"
+	"time"
+
+	packages_model "code.gitea.io/gitea/models/packages"
+	container_model "code.gitea.io/gitea/models/packages/container"
+)
+
+// Cleanup removes expired container data
+func Cleanup(ctx context.Context, olderThan time.Duration) error {
+	if err := cleanupExpiredBlobUploads(ctx, olderThan); err != nil {
+		return err
+	}
+	return cleanupExpiredUploadedBlobs(ctx, olderThan)
+}
+
+// cleanupExpiredBlobUploads removes expired blob uploads
+func cleanupExpiredBlobUploads(ctx context.Context, olderThan time.Duration) error {
+	pbus, err := packages_model.FindExpiredBlobUploads(ctx, olderThan)
+	if err != nil {
+		return err
+	}
+
+	for _, pbu := range pbus {
+		if err := RemoveBlobUploadByID(ctx, pbu.ID); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// cleanupExpiredUploadedBlobs removes expired uploaded blobs not referenced by a manifest
+func cleanupExpiredUploadedBlobs(ctx context.Context, olderThan time.Duration) error {
+	pfs, err := container_model.SearchExpiredUploadedBlobs(ctx, olderThan)
+	if err != nil {
+		return err
+	}
+
+	versions := make(map[int64]struct{})
+	for _, pf := range pfs {
+		versions[pf.VersionID] = struct{}{}
+
+		if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil {
+			return err
+		}
+		if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil {
+			return err
+		}
+	}
+
+	for versionID := range versions {
+		has, err := packages_model.HasVersionFileReferences(ctx, versionID)
+		if err != nil {
+			return err
+		}
+		if !has {
+			if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeVersion, versionID); err != nil {
+				return err
+			}
+
+			if err := packages_model.DeleteVersionByID(ctx, versionID); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
diff --git a/services/packages/packages.go b/services/packages/packages.go
new file mode 100644
index 0000000000..b26e60c711
--- /dev/null
+++ b/services/packages/packages.go
@@ -0,0 +1,458 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package packages
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"code.gitea.io/gitea/models/db"
+	packages_model "code.gitea.io/gitea/models/packages"
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/json"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/notification"
+	packages_module "code.gitea.io/gitea/modules/packages"
+	container_service "code.gitea.io/gitea/services/packages/container"
+)
+
+// PackageInfo describes a package
+type PackageInfo struct {
+	Owner       *user_model.User
+	PackageType packages_model.Type
+	Name        string
+	Version     string
+}
+
+// PackageCreationInfo describes a package to create
+type PackageCreationInfo struct {
+	PackageInfo
+	SemverCompatible bool
+	Creator          *user_model.User
+	Metadata         interface{}
+	Properties       map[string]string
+}
+
+// PackageFileInfo describes a package file
+type PackageFileInfo struct {
+	Filename     string
+	CompositeKey string
+}
+
+// PackageFileCreationInfo describes a package file to create
+type PackageFileCreationInfo struct {
+	PackageFileInfo
+	Data              packages_module.HashedSizeReader
+	IsLead            bool
+	Properties        map[string]string
+	OverwriteExisting bool
+}
+
+// CreatePackageAndAddFile creates a package with a file. If the same package exists already, ErrDuplicatePackageVersion is returned
+func CreatePackageAndAddFile(pvci *PackageCreationInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageVersion, *packages_model.PackageFile, error) {
+	return createPackageAndAddFile(pvci, pfci, false)
+}
+
+// CreatePackageOrAddFileToExisting creates a package with a file or adds the file if the package exists already
+func CreatePackageOrAddFileToExisting(pvci *PackageCreationInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageVersion, *packages_model.PackageFile, error) {
+	return createPackageAndAddFile(pvci, pfci, true)
+}
+
+func createPackageAndAddFile(pvci *PackageCreationInfo, pfci *PackageFileCreationInfo, allowDuplicate bool) (*packages_model.PackageVersion, *packages_model.PackageFile, error) {
+	ctx, committer, err := db.TxContext()
+	if err != nil {
+		return nil, nil, err
+	}
+	defer committer.Close()
+
+	pv, created, err := createPackageAndVersion(ctx, pvci, allowDuplicate)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pfci)
+	removeBlob := false
+	defer func() {
+		if blobCreated && removeBlob {
+			contentStore := packages_module.NewContentStore()
+			if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
+				log.Error("Error deleting package blob from content store: %v", err)
+			}
+		}
+	}()
+	if err != nil {
+		removeBlob = true
+		return nil, nil, err
+	}
+
+	if err := committer.Commit(); err != nil {
+		removeBlob = true
+		return nil, nil, err
+	}
+
+	if created {
+		pd, err := packages_model.GetPackageDescriptor(ctx, pv)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		notification.NotifyPackageCreate(pvci.Creator, pd)
+	}
+
+	return pv, pf, nil
+}
+
+func createPackageAndVersion(ctx context.Context, pvci *PackageCreationInfo, allowDuplicate bool) (*packages_model.PackageVersion, bool, error) {
+	log.Trace("Creating package: %v, %v, %v, %s, %s, %+v, %v", pvci.Creator.ID, pvci.Owner.ID, pvci.PackageType, pvci.Name, pvci.Version, pvci.Properties, allowDuplicate)
+
+	p := &packages_model.Package{
+		OwnerID:          pvci.Owner.ID,
+		Type:             pvci.PackageType,
+		Name:             pvci.Name,
+		LowerName:        strings.ToLower(pvci.Name),
+		SemverCompatible: pvci.SemverCompatible,
+	}
+	var err error
+	if p, err = packages_model.TryInsertPackage(ctx, p); err != nil {
+		if err != packages_model.ErrDuplicatePackage {
+			log.Error("Error inserting package: %v", err)
+			return nil, false, err
+		}
+	}
+
+	metadataJSON, err := json.Marshal(pvci.Metadata)
+	if err != nil {
+		return nil, false, err
+	}
+
+	created := true
+	pv := &packages_model.PackageVersion{
+		PackageID:    p.ID,
+		CreatorID:    pvci.Creator.ID,
+		Version:      pvci.Version,
+		LowerVersion: strings.ToLower(pvci.Version),
+		MetadataJSON: string(metadataJSON),
+	}
+	if pv, err = packages_model.GetOrInsertVersion(ctx, pv); err != nil {
+		if err == packages_model.ErrDuplicatePackageVersion {
+			created = false
+		}
+		if err != packages_model.ErrDuplicatePackageVersion || !allowDuplicate {
+			log.Error("Error inserting package: %v", err)
+			return nil, false, err
+		}
+	}
+
+	if created {
+		for name, value := range pvci.Properties {
+			if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, name, value); err != nil {
+				log.Error("Error setting package version property: %v", err)
+				return nil, false, err
+			}
+		}
+	}
+
+	return pv, created, nil
+}
+
+// AddFileToExistingPackage adds a file to an existing package. If the package does not exist, ErrPackageNotExist is returned
+func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageVersion, *packages_model.PackageFile, error) {
+	ctx, committer, err := db.TxContext()
+	if err != nil {
+		return nil, nil, err
+	}
+	defer committer.Close()
+
+	pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pfci)
+	removeBlob := false
+	defer func() {
+		if removeBlob {
+			contentStore := packages_module.NewContentStore()
+			if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
+				log.Error("Error deleting package blob from content store: %v", err)
+			}
+		}
+	}()
+	if err != nil {
+		removeBlob = blobCreated
+		return nil, nil, err
+	}
+
+	if err := committer.Commit(); err != nil {
+		removeBlob = blobCreated
+		return nil, nil, err
+	}
+
+	return pv, pf, nil
+}
+
+// NewPackageBlob creates a package blob instance
+func NewPackageBlob(hsr packages_module.HashedSizeReader) *packages_model.PackageBlob {
+	hashMD5, hashSHA1, hashSHA256, hashSHA512 := hsr.Sums()
+
+	return &packages_model.PackageBlob{
+		Size:       hsr.Size(),
+		HashMD5:    fmt.Sprintf("%x", hashMD5),
+		HashSHA1:   fmt.Sprintf("%x", hashSHA1),
+		HashSHA256: fmt.Sprintf("%x", hashSHA256),
+		HashSHA512: fmt.Sprintf("%x", hashSHA512),
+	}
+}
+
+func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) {
+	log.Trace("Adding package file: %v, %s", pv.ID, pfci.Filename)
+
+	pb, exists, err := packages_model.GetOrInsertBlob(ctx, NewPackageBlob(pfci.Data))
+	if err != nil {
+		log.Error("Error inserting package blob: %v", err)
+		return nil, nil, false, err
+	}
+	if !exists {
+		contentStore := packages_module.NewContentStore()
+		if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), pfci.Data, pfci.Data.Size()); err != nil {
+			log.Error("Error saving package blob in content store: %v", err)
+			return nil, nil, false, err
+		}
+	}
+
+	if pfci.OverwriteExisting {
+		pf, err := packages_model.GetFileForVersionByName(ctx, pv.ID, pfci.Filename, pfci.CompositeKey)
+		if err != nil && err != packages_model.ErrPackageFileNotExist {
+			return nil, pb, !exists, err
+		}
+		if pf != nil {
+			// Short circuit if blob is the same
+			if pf.BlobID == pb.ID {
+				return pf, pb, !exists, nil
+			}
+
+			if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil {
+				return nil, pb, !exists, err
+			}
+			if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil {
+				return nil, pb, !exists, err
+			}
+		}
+	}
+
+	pf := &packages_model.PackageFile{
+		VersionID:    pv.ID,
+		BlobID:       pb.ID,
+		Name:         pfci.Filename,
+		LowerName:    strings.ToLower(pfci.Filename),
+		CompositeKey: pfci.CompositeKey,
+		IsLead:       pfci.IsLead,
+	}
+	if pf, err = packages_model.TryInsertFile(ctx, pf); err != nil {
+		if err != packages_model.ErrDuplicatePackageFile {
+			log.Error("Error inserting package file: %v", err)
+		}
+		return nil, pb, !exists, err
+	}
+
+	for name, value := range pfci.Properties {
+		if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeFile, pf.ID, name, value); err != nil {
+			log.Error("Error setting package file property: %v", err)
+			return pf, pb, !exists, err
+		}
+	}
+
+	return pf, pb, !exists, nil
+}
+
+// RemovePackageVersionByNameAndVersion deletes a package version and all associated files
+func RemovePackageVersionByNameAndVersion(doer *user_model.User, pvi *PackageInfo) error {
+	pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version)
+	if err != nil {
+		return err
+	}
+
+	return RemovePackageVersion(doer, pv)
+}
+
+// RemovePackageVersion deletes the package version and all associated files
+func RemovePackageVersion(doer *user_model.User, pv *packages_model.PackageVersion) error {
+	ctx, committer, err := db.TxContext()
+	if err != nil {
+		return err
+	}
+	defer committer.Close()
+
+	pd, err := packages_model.GetPackageDescriptor(ctx, pv)
+	if err != nil {
+		return err
+	}
+
+	log.Trace("Deleting package: %v", pv.ID)
+
+	if err := DeletePackageVersionAndReferences(ctx, pv); err != nil {
+		return err
+	}
+
+	if err := committer.Commit(); err != nil {
+		return err
+	}
+
+	notification.NotifyPackageDelete(doer, pd)
+
+	return nil
+}
+
+// DeletePackageVersionAndReferences deletes the package version and its properties and files
+func DeletePackageVersionAndReferences(ctx context.Context, pv *packages_model.PackageVersion) error {
+	if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeVersion, pv.ID); err != nil {
+		return err
+	}
+
+	pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID)
+	if err != nil {
+		return err
+	}
+
+	for _, pf := range pfs {
+		if err := DeletePackageFile(ctx, pf); err != nil {
+			return err
+		}
+	}
+
+	return packages_model.DeleteVersionByID(ctx, pv.ID)
+}
+
+// DeletePackageFile deletes the package file and its properties
+func DeletePackageFile(ctx context.Context, pf *packages_model.PackageFile) error {
+	if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil {
+		return err
+	}
+	return packages_model.DeleteFileByID(ctx, pf.ID)
+}
+
+// Cleanup removes old unreferenced package blobs
+func Cleanup(unused context.Context, olderThan time.Duration) error {
+	ctx, committer, err := db.TxContext()
+	if err != nil {
+		return err
+	}
+	defer committer.Close()
+
+	if err := container_service.Cleanup(ctx, olderThan); err != nil {
+		log.Error("hier")
+		return err
+	}
+
+	if err := packages_model.DeletePackagesIfUnreferenced(ctx); err != nil {
+		log.Error("hier2")
+		return err
+	}
+
+	pbs, err := packages_model.FindExpiredUnreferencedBlobs(ctx, olderThan)
+	if err != nil {
+		log.Error("hier3")
+		return err
+	}
+
+	for _, pb := range pbs {
+		if err := packages_model.DeleteBlobByID(ctx, pb.ID); err != nil {
+			log.Error("hier4")
+			return err
+		}
+	}
+
+	if err := committer.Commit(); err != nil {
+		return err
+	}
+
+	contentStore := packages_module.NewContentStore()
+	for _, pb := range pbs {
+		if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
+			log.Error("Error deleting package blob [%v]: %v", pb.ID, err)
+		}
+	}
+
+	return nil
+}
+
+// GetFileStreamByPackageNameAndVersion returns the content of the specific package file
+func GetFileStreamByPackageNameAndVersion(ctx context.Context, pvi *PackageInfo, pfi *PackageFileInfo) (io.ReadCloser, *packages_model.PackageFile, error) {
+	log.Trace("Getting package file stream: %v, %v, %s, %s, %s, %s", pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, pfi.Filename, pfi.CompositeKey)
+
+	pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version)
+	if err != nil {
+		if err == packages_model.ErrPackageNotExist {
+			return nil, nil, err
+		}
+		log.Error("Error getting package: %v", err)
+		return nil, nil, err
+	}
+
+	return GetFileStreamByPackageVersion(ctx, pv, pfi)
+}
+
+// GetFileStreamByPackageVersionAndFileID returns the content of the specific package file
+func GetFileStreamByPackageVersionAndFileID(ctx context.Context, owner *user_model.User, versionID, fileID int64) (io.ReadCloser, *packages_model.PackageFile, error) {
+	log.Trace("Getting package file stream: %v, %v, %v", owner.ID, versionID, fileID)
+
+	pv, err := packages_model.GetVersionByID(ctx, versionID)
+	if err != nil {
+		if err == packages_model.ErrPackageVersionNotExist {
+			return nil, nil, packages_model.ErrPackageNotExist
+		}
+		log.Error("Error getting package version: %v", err)
+		return nil, nil, err
+	}
+
+	p, err := packages_model.GetPackageByID(ctx, pv.PackageID)
+	if err != nil {
+		log.Error("Error getting package: %v", err)
+		return nil, nil, err
+	}
+
+	if p.OwnerID != owner.ID {
+		return nil, nil, packages_model.ErrPackageNotExist
+	}
+
+	pf, err := packages_model.GetFileForVersionByID(ctx, versionID, fileID)
+	if err != nil {
+		log.Error("Error getting file: %v", err)
+		return nil, nil, err
+	}
+
+	return GetPackageFileStream(ctx, pv, pf)
+}
+
+// GetFileStreamByPackageVersion returns the content of the specific package file
+func GetFileStreamByPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pfi *PackageFileInfo) (io.ReadCloser, *packages_model.PackageFile, error) {
+	pf, err := packages_model.GetFileForVersionByName(db.DefaultContext, pv.ID, pfi.Filename, pfi.CompositeKey)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return GetPackageFileStream(ctx, pv, pf)
+}
+
+// GetPackageFileStream returns the content of the specific package file
+func GetPackageFileStream(ctx context.Context, pv *packages_model.PackageVersion, pf *packages_model.PackageFile) (io.ReadCloser, *packages_model.PackageFile, error) {
+	pb, err := packages_model.GetBlobByID(ctx, pf.BlobID)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	s, err := packages_module.NewContentStore().Get(packages_module.BlobHash256Key(pb.HashSHA256))
+	if err == nil {
+		if pf.IsLead {
+			if err := packages_model.IncrementDownloadCounter(ctx, pv.ID); err != nil {
+				log.Error("Error incrementing download counter: %v", err)
+			}
+		}
+	}
+	return s, pf, err
+}