aboutsummaryrefslogtreecommitdiffstats
path: root/services
diff options
context:
space:
mode:
Diffstat (limited to 'services')
-rw-r--r--services/forms/package_form.go2
-rw-r--r--services/packages/cleanup/cleanup.go11
-rw-r--r--services/packages/debian/repository.go443
-rw-r--r--services/packages/packages.go46
4 files changed, 488 insertions, 14 deletions
diff --git a/services/forms/package_form.go b/services/forms/package_form.go
index 699d0fe44f..efe4a0d84a 100644
--- a/services/forms/package_form.go
+++ b/services/forms/package_form.go
@@ -15,7 +15,7 @@ import (
type PackageCleanupRuleForm struct {
ID int64
Enabled bool
- Type string `binding:"Required;In(cargo,chef,composer,conan,conda,container,generic,helm,maven,npm,nuget,pub,pypi,rubygems,swift,vagrant)"`
+ Type string `binding:"Required;In(cargo,chef,composer,conan,conda,container,debian,generic,helm,maven,npm,nuget,pub,pypi,rubygems,swift,vagrant)"`
KeepCount int `binding:"In(0,1,5,10,25,50,100)"`
KeepPattern string `binding:"RegexPattern"`
RemoveDays int `binding:"In(0,7,14,30,60,90,180)"`
diff --git a/services/packages/cleanup/cleanup.go b/services/packages/cleanup/cleanup.go
index 2d62a028a4..43fbc1ad9b 100644
--- a/services/packages/cleanup/cleanup.go
+++ b/services/packages/cleanup/cleanup.go
@@ -17,6 +17,7 @@ import (
packages_service "code.gitea.io/gitea/services/packages"
cargo_service "code.gitea.io/gitea/services/packages/cargo"
container_service "code.gitea.io/gitea/services/packages/container"
+ debian_service "code.gitea.io/gitea/services/packages/debian"
)
// Cleanup removes expired package data
@@ -45,6 +46,7 @@ func Cleanup(taskCtx context.Context, olderThan time.Duration) error {
return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err)
}
+ anyVersionDeleted := false
for _, p := range packages {
pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
PackageID: p.ID,
@@ -91,6 +93,7 @@ func Cleanup(taskCtx context.Context, olderThan time.Duration) error {
}
versionDeleted = true
+ anyVersionDeleted = true
}
if versionDeleted {
@@ -105,6 +108,14 @@ func Cleanup(taskCtx context.Context, olderThan time.Duration) error {
}
}
}
+
+ if anyVersionDeleted {
+ if pcr.Type == packages_model.TypeDebian {
+ if err := debian_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil {
+ return fmt.Errorf("CleanupRule [%d]: debian.BuildAllRepositoryFiles failed: %w", pcr.ID, err)
+ }
+ }
+ }
return nil
})
if err != nil {
diff --git a/services/packages/debian/repository.go b/services/packages/debian/repository.go
new file mode 100644
index 0000000000..eac878256a
--- /dev/null
+++ b/services/packages/debian/repository.go
@@ -0,0 +1,443 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package debian
+
+import (
+ "bytes"
+ "compress/gzip"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "sort"
+ "strings"
+ "time"
+
+ "code.gitea.io/gitea/models/db"
+ packages_model "code.gitea.io/gitea/models/packages"
+ debian_model "code.gitea.io/gitea/models/packages/debian"
+ user_model "code.gitea.io/gitea/models/user"
+ "code.gitea.io/gitea/modules/log"
+ packages_module "code.gitea.io/gitea/modules/packages"
+ debian_module "code.gitea.io/gitea/modules/packages/debian"
+ "code.gitea.io/gitea/modules/setting"
+ "code.gitea.io/gitea/modules/util"
+ packages_service "code.gitea.io/gitea/services/packages"
+
+ "github.com/keybase/go-crypto/openpgp"
+ "github.com/keybase/go-crypto/openpgp/armor"
+ "github.com/keybase/go-crypto/openpgp/clearsign"
+ "github.com/keybase/go-crypto/openpgp/packet"
+ "github.com/ulikunitz/xz"
+)
+
+// GetOrCreateRepositoryVersion gets or creates the internal repository package
+// The Debian registry needs multiple index files which are stored in this package.
+func GetOrCreateRepositoryVersion(ownerID int64) (*packages_model.PackageVersion, error) {
+ var repositoryVersion *packages_model.PackageVersion
+
+ return repositoryVersion, db.WithTx(db.DefaultContext, func(ctx context.Context) error {
+ p := &packages_model.Package{
+ OwnerID: ownerID,
+ Type: packages_model.TypeDebian,
+ Name: debian_module.RepositoryPackage,
+ LowerName: debian_module.RepositoryPackage,
+ IsInternal: true,
+ }
+ var err error
+ if p, err = packages_model.TryInsertPackage(ctx, p); err != nil {
+ if err != packages_model.ErrDuplicatePackage {
+ log.Error("Error inserting package: %v", err)
+ return err
+ }
+ }
+
+ pv := &packages_model.PackageVersion{
+ PackageID: p.ID,
+ CreatorID: ownerID,
+ Version: debian_module.RepositoryVersion,
+ LowerVersion: debian_module.RepositoryVersion,
+ IsInternal: true,
+ MetadataJSON: "null",
+ }
+ if pv, err = packages_model.GetOrInsertVersion(ctx, pv); err != nil {
+ if err != packages_model.ErrDuplicatePackageVersion {
+ log.Error("Error inserting package version: %v", err)
+ return err
+ }
+ }
+
+ repositoryVersion = pv
+
+ return nil
+ })
+}
+
+// GetOrCreateKeyPair gets or creates the PGP keys used to sign repository files
+func GetOrCreateKeyPair(ownerID int64) (string, string, error) {
+ priv, err := user_model.GetSetting(ownerID, debian_module.SettingKeyPrivate)
+ if err != nil && !errors.Is(err, util.ErrNotExist) {
+ return "", "", err
+ }
+
+ pub, err := user_model.GetSetting(ownerID, debian_module.SettingKeyPublic)
+ if err != nil && !errors.Is(err, util.ErrNotExist) {
+ return "", "", err
+ }
+
+ if priv == "" || pub == "" {
+ priv, pub, err = generateKeypair()
+ if err != nil {
+ return "", "", err
+ }
+
+ if err := user_model.SetUserSetting(ownerID, debian_module.SettingKeyPrivate, priv); err != nil {
+ return "", "", err
+ }
+
+ if err := user_model.SetUserSetting(ownerID, debian_module.SettingKeyPublic, pub); err != nil {
+ return "", "", err
+ }
+ }
+
+ return priv, pub, nil
+}
+
+func generateKeypair() (string, string, error) {
+ e, err := openpgp.NewEntity(setting.AppName, "Debian Registry", "", nil)
+ if err != nil {
+ return "", "", err
+ }
+
+ var priv strings.Builder
+ var pub strings.Builder
+
+ w, err := armor.Encode(&priv, openpgp.PrivateKeyType, nil)
+ if err != nil {
+ return "", "", err
+ }
+ if err := e.SerializePrivate(w, nil); err != nil {
+ return "", "", err
+ }
+ w.Close()
+
+ w, err = armor.Encode(&pub, openpgp.PublicKeyType, nil)
+ if err != nil {
+ return "", "", err
+ }
+ if err := e.Serialize(w); err != nil {
+ return "", "", err
+ }
+ w.Close()
+
+ return priv.String(), pub.String(), nil
+}
+
+// BuildAllRepositoryFiles (re)builds all repository files for every available distributions, components and architectures
+func BuildAllRepositoryFiles(ctx context.Context, ownerID int64) error {
+ pv, err := GetOrCreateRepositoryVersion(ownerID)
+ if err != nil {
+ return err
+ }
+
+ // 1. Delete all existing repository files
+ pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID)
+ if err != nil {
+ return err
+ }
+
+ for _, pf := range pfs {
+ if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil {
+ return err
+ }
+ if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil {
+ return err
+ }
+ }
+
+ // 2. (Re)Build repository files for existing packages
+ distributions, err := debian_model.GetDistributions(ctx, ownerID)
+ if err != nil {
+ return err
+ }
+ for _, distribution := range distributions {
+ components, err := debian_model.GetComponents(ctx, ownerID, distribution)
+ if err != nil {
+ return err
+ }
+ architectures, err := debian_model.GetArchitectures(ctx, ownerID, distribution)
+ if err != nil {
+ return err
+ }
+
+ for _, component := range components {
+ for _, architecture := range architectures {
+ if err := buildRepositoryFiles(ctx, ownerID, pv, distribution, component, architecture); err != nil {
+ return fmt.Errorf("failed to build repository files [%s/%s/%s]: %w", distribution, component, architecture, err)
+ }
+ }
+ }
+ }
+
+ return nil
+}
+
+// BuildSpecificRepositoryFiles builds index files for the repository
+func BuildSpecificRepositoryFiles(ctx context.Context, ownerID int64, distribution, component, architecture string) error {
+ pv, err := GetOrCreateRepositoryVersion(ownerID)
+ if err != nil {
+ return err
+ }
+
+ return buildRepositoryFiles(ctx, ownerID, pv, distribution, component, architecture)
+}
+
+func buildRepositoryFiles(ctx context.Context, ownerID int64, repoVersion *packages_model.PackageVersion, distribution, component, architecture string) error {
+ if err := buildPackagesIndices(ctx, ownerID, repoVersion, distribution, component, architecture); err != nil {
+ return err
+ }
+
+ return buildReleaseFiles(ctx, ownerID, repoVersion, distribution)
+}
+
+// https://wiki.debian.org/DebianRepository/Format#A.22Packages.22_Indices
+func buildPackagesIndices(ctx context.Context, ownerID int64, repoVersion *packages_model.PackageVersion, distribution, component, architecture string) error {
+ pfds, err := debian_model.SearchLatestPackages(ctx, &debian_model.PackageSearchOptions{
+ OwnerID: ownerID,
+ Distribution: distribution,
+ Component: component,
+ Architecture: architecture,
+ })
+ if err != nil {
+ return err
+ }
+
+ // Delete the package indices if there are no packages
+ if len(pfds) == 0 {
+ key := fmt.Sprintf("%s|%s|%s", distribution, component, architecture)
+ for _, filename := range []string{"Packages", "Packages.gz", "Packages.xz"} {
+ pf, err := packages_model.GetFileForVersionByName(ctx, repoVersion.ID, filename, key)
+ if err != nil && !errors.Is(err, util.ErrNotExist) {
+ return err
+ }
+
+ if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil {
+ return err
+ }
+ if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil {
+ return err
+ }
+ }
+
+ return nil
+ }
+
+ packagesContent, _ := packages_module.NewHashedBuffer()
+
+ packagesGzipContent, _ := packages_module.NewHashedBuffer()
+ gzw := gzip.NewWriter(packagesGzipContent)
+
+ packagesXzContent, _ := packages_module.NewHashedBuffer()
+ xzw, _ := xz.NewWriter(packagesXzContent)
+
+ w := io.MultiWriter(packagesContent, gzw, xzw)
+
+ addSeperator := false
+ for _, pfd := range pfds {
+ if addSeperator {
+ fmt.Fprintln(w)
+ }
+ addSeperator = true
+
+ fmt.Fprint(w, pfd.Properties.GetByName(debian_module.PropertyControl))
+
+ fmt.Fprintf(w, "Filename: pool/%s/%s/%s\n", distribution, component, pfd.File.Name)
+ fmt.Fprintf(w, "Size: %d\n", pfd.Blob.Size)
+ fmt.Fprintf(w, "MD5sum: %s\n", pfd.Blob.HashMD5)
+ fmt.Fprintf(w, "SHA1: %s\n", pfd.Blob.HashSHA1)
+ fmt.Fprintf(w, "SHA256: %s\n", pfd.Blob.HashSHA256)
+ fmt.Fprintf(w, "SHA512: %s\n", pfd.Blob.HashSHA512)
+ }
+
+ gzw.Close()
+ xzw.Close()
+
+ for _, file := range []struct {
+ Name string
+ Data packages_module.HashedSizeReader
+ }{
+ {"Packages", packagesContent},
+ {"Packages.gz", packagesGzipContent},
+ {"Packages.xz", packagesXzContent},
+ } {
+ _, err = packages_service.AddFileToPackageVersionInternal(
+ repoVersion,
+ &packages_service.PackageFileCreationInfo{
+ PackageFileInfo: packages_service.PackageFileInfo{
+ Filename: file.Name,
+ CompositeKey: fmt.Sprintf("%s|%s|%s", distribution, component, architecture),
+ },
+ Creator: user_model.NewGhostUser(),
+ Data: file.Data,
+ IsLead: false,
+ OverwriteExisting: true,
+ Properties: map[string]string{
+ debian_module.PropertyRepositoryIncludeInRelease: "",
+ debian_module.PropertyDistribution: distribution,
+ debian_module.PropertyComponent: component,
+ debian_module.PropertyArchitecture: architecture,
+ },
+ },
+ )
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+// https://wiki.debian.org/DebianRepository/Format#A.22Release.22_files
+func buildReleaseFiles(ctx context.Context, ownerID int64, repoVersion *packages_model.PackageVersion, distribution string) error {
+ pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{
+ VersionID: repoVersion.ID,
+ Properties: map[string]string{
+ debian_module.PropertyRepositoryIncludeInRelease: "",
+ debian_module.PropertyDistribution: distribution,
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ // Delete the release files if there are no packages
+ if len(pfs) == 0 {
+ for _, filename := range []string{"Release", "Release.gpg", "InRelease"} {
+ pf, err := packages_model.GetFileForVersionByName(ctx, repoVersion.ID, filename, distribution)
+ if err != nil && !errors.Is(err, util.ErrNotExist) {
+ return err
+ }
+
+ if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil {
+ return err
+ }
+ if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil {
+ return err
+ }
+ }
+
+ return nil
+ }
+
+ components, err := debian_model.GetComponents(ctx, ownerID, distribution)
+ if err != nil {
+ return err
+ }
+
+ sort.Strings(components)
+
+ architectures, err := debian_model.GetArchitectures(ctx, ownerID, distribution)
+ if err != nil {
+ return err
+ }
+
+ sort.Strings(architectures)
+
+ priv, _, err := GetOrCreateKeyPair(ownerID)
+ if err != nil {
+ return err
+ }
+
+ block, err := armor.Decode(strings.NewReader(priv))
+ if err != nil {
+ return err
+ }
+
+ e, err := openpgp.ReadEntity(packet.NewReader(block.Body))
+ if err != nil {
+ return err
+ }
+
+ inReleaseContent, _ := packages_module.NewHashedBuffer()
+ sw, err := clearsign.Encode(inReleaseContent, e.PrivateKey, nil)
+ if err != nil {
+ return err
+ }
+
+ var buf bytes.Buffer
+
+ w := io.MultiWriter(sw, &buf)
+
+ fmt.Fprintf(w, "Origin: %s\n", setting.AppName)
+ fmt.Fprintf(w, "Label: %s\n", setting.AppName)
+ fmt.Fprintf(w, "Suite: %s\n", distribution)
+ fmt.Fprintf(w, "Codename: %s\n", distribution)
+ fmt.Fprintf(w, "Components: %s\n", strings.Join(components, " "))
+ fmt.Fprintf(w, "Architectures: %s\n", strings.Join(architectures, " "))
+ fmt.Fprintf(w, "Date: %s\n", time.Now().UTC().Format(time.RFC1123))
+ fmt.Fprint(w, "Acquire-By-Hash: yes")
+
+ pfds, err := packages_model.GetPackageFileDescriptors(ctx, pfs)
+ if err != nil {
+ return err
+ }
+
+ var md5, sha1, sha256, sha512 strings.Builder
+ for _, pfd := range pfds {
+ path := fmt.Sprintf("%s/binary-%s/%s", pfd.Properties.GetByName(debian_module.PropertyComponent), pfd.Properties.GetByName(debian_module.PropertyArchitecture), pfd.File.Name)
+ fmt.Fprintf(&md5, " %s %d %s\n", pfd.Blob.HashMD5, pfd.Blob.Size, path)
+ fmt.Fprintf(&sha1, " %s %d %s\n", pfd.Blob.HashSHA1, pfd.Blob.Size, path)
+ fmt.Fprintf(&sha256, " %s %d %s\n", pfd.Blob.HashSHA256, pfd.Blob.Size, path)
+ fmt.Fprintf(&sha512, " %s %d %s\n", pfd.Blob.HashSHA512, pfd.Blob.Size, path)
+ }
+
+ fmt.Fprintln(w, "MD5Sum:")
+ fmt.Fprint(w, md5.String())
+ fmt.Fprintln(w, "SHA1:")
+ fmt.Fprint(w, sha1.String())
+ fmt.Fprintln(w, "SHA256:")
+ fmt.Fprint(w, sha256.String())
+ fmt.Fprintln(w, "SHA512:")
+ fmt.Fprint(w, sha512.String())
+
+ sw.Close()
+
+ releaseGpgContent, _ := packages_module.NewHashedBuffer()
+ if err := openpgp.ArmoredDetachSign(releaseGpgContent, e, bytes.NewReader(buf.Bytes()), nil); err != nil {
+ return err
+ }
+
+ releaseContent, _ := packages_module.CreateHashedBufferFromReader(&buf)
+
+ for _, file := range []struct {
+ Name string
+ Data packages_module.HashedSizeReader
+ }{
+ {"Release", releaseContent},
+ {"Release.gpg", releaseGpgContent},
+ {"InRelease", inReleaseContent},
+ } {
+ _, err = packages_service.AddFileToPackageVersionInternal(
+ repoVersion,
+ &packages_service.PackageFileCreationInfo{
+ PackageFileInfo: packages_service.PackageFileInfo{
+ Filename: file.Name,
+ CompositeKey: distribution,
+ },
+ Creator: user_model.NewGhostUser(),
+ Data: file.Data,
+ IsLead: false,
+ OverwriteExisting: true,
+ Properties: map[string]string{
+ debian_module.PropertyDistribution: distribution,
+ },
+ },
+ )
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
diff --git a/services/packages/packages.go b/services/packages/packages.go
index dd5c63470b..735e52c854 100644
--- a/services/packages/packages.go
+++ b/services/packages/packages.go
@@ -187,19 +187,33 @@ func createPackageAndVersion(ctx context.Context, pvci *PackageCreationInfo, all
}
// AddFileToExistingPackage adds a file to an existing package. If the package does not exist, ErrPackageNotExist is returned
-func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageVersion, *packages_model.PackageFile, error) {
+func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, error) {
+ return addFileToPackageWrapper(func(ctx context.Context) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) {
+ pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version)
+ if err != nil {
+ return nil, nil, false, err
+ }
+
+ return addFileToPackageVersion(ctx, pv, pvi, pfci)
+ })
+}
+
+// AddFileToPackageVersionInternal adds a file to the package
+// This method skips quota checks and should only be used for system-managed packages.
+func AddFileToPackageVersionInternal(pv *packages_model.PackageVersion, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, error) {
+ return addFileToPackageWrapper(func(ctx context.Context) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) {
+ return addFileToPackageVersionUnchecked(ctx, pv, pfci)
+ })
+}
+
+func addFileToPackageWrapper(fn func(ctx context.Context) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error)) (*packages_model.PackageFile, error) {
ctx, committer, err := db.TxContext(db.DefaultContext)
if err != nil {
- return nil, nil, err
+ return nil, err
}
defer committer.Close()
- pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version)
- if err != nil {
- return nil, nil, err
- }
-
- pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pvi, pfci)
+ pf, pb, blobCreated, err := fn(ctx)
removeBlob := false
defer func() {
if removeBlob {
@@ -211,15 +225,15 @@ func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) (
}()
if err != nil {
removeBlob = blobCreated
- return nil, nil, err
+ return nil, err
}
if err := committer.Commit(); err != nil {
removeBlob = blobCreated
- return nil, nil, err
+ return nil, err
}
- return pv, pf, nil
+ return pf, nil
}
// NewPackageBlob creates a package blob instance
@@ -236,12 +250,16 @@ func NewPackageBlob(hsr packages_module.HashedSizeReader) *packages_model.Packag
}
func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pvi *PackageInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) {
- log.Trace("Adding package file: %v, %s", pv.ID, pfci.Filename)
-
if err := CheckSizeQuotaExceeded(ctx, pfci.Creator, pvi.Owner, pvi.PackageType, pfci.Data.Size()); err != nil {
return nil, nil, false, err
}
+ return addFileToPackageVersionUnchecked(ctx, pv, pfci)
+}
+
+func addFileToPackageVersionUnchecked(ctx context.Context, pv *packages_model.PackageVersion, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) {
+ log.Trace("Adding package file: %v, %s", pv.ID, pfci.Filename)
+
pb, exists, err := packages_model.GetOrInsertBlob(ctx, NewPackageBlob(pfci.Data))
if err != nil {
log.Error("Error inserting package blob: %v", err)
@@ -345,6 +363,8 @@ func CheckSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, p
typeSpecificSize = setting.Packages.LimitSizeConda
case packages_model.TypeContainer:
typeSpecificSize = setting.Packages.LimitSizeContainer
+ case packages_model.TypeDebian:
+ typeSpecificSize = setting.Packages.LimitSizeDebian
case packages_model.TypeGeneric:
typeSpecificSize = setting.Packages.LimitSizeGeneric
case packages_model.TypeHelm:
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-26 15:26:05 +0000