Add Debian package registry (#22854)

Co-authored-by: @awkwardbunny

This PR adds a Debian package registry. You can follow [this
tutorial](https://www.baeldung.com/linux/create-debian-package) to build
a *.deb package for testing. Source packages are not supported at the
moment and I did not find documentation of the architecture "all" and
how these packages should be treated.

---------

Co-authored-by: Brian Hong <brian@hongs.me>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>

4 files changed, 488 insertions, 14 deletions

diff --git a/services/forms/package_form.go b/services/forms/package_form.go
index 699d0fe44f..efe4a0d84a 100644
--- a/services/forms/package_form.go
+++ b/services/forms/package_form.go
@@ -15,7 +15,7 @@ import (
 type PackageCleanupRuleForm struct {
 	ID            int64
 	Enabled       bool
-	Type          string `binding:"Required;In(cargo,chef,composer,conan,conda,container,generic,helm,maven,npm,nuget,pub,pypi,rubygems,swift,vagrant)"`
+	Type          string `binding:"Required;In(cargo,chef,composer,conan,conda,container,debian,generic,helm,maven,npm,nuget,pub,pypi,rubygems,swift,vagrant)"`
 	KeepCount     int    `binding:"In(0,1,5,10,25,50,100)"`
 	KeepPattern   string `binding:"RegexPattern"`
 	RemoveDays    int    `binding:"In(0,7,14,30,60,90,180)"`
diff --git a/services/packages/cleanup/cleanup.go b/services/packages/cleanup/cleanup.go
index 2d62a028a4..43fbc1ad9b 100644
--- a/services/packages/cleanup/cleanup.go
+++ b/services/packages/cleanup/cleanup.go
@@ -17,6 +17,7 @@ import (
 	packages_service "code.gitea.io/gitea/services/packages"
 	cargo_service "code.gitea.io/gitea/services/packages/cargo"
 	container_service "code.gitea.io/gitea/services/packages/container"
+	debian_service "code.gitea.io/gitea/services/packages/debian"
 )
 
 // Cleanup removes expired package data
@@ -45,6 +46,7 @@ func Cleanup(taskCtx context.Context, olderThan time.Duration) error {
 			return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err)
 		}
 
+		anyVersionDeleted := false
 		for _, p := range packages {
 			pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
 				PackageID:  p.ID,
@@ -91,6 +93,7 @@ func Cleanup(taskCtx context.Context, olderThan time.Duration) error {
 				}
 
 				versionDeleted = true
+				anyVersionDeleted = true
 			}
 
 			if versionDeleted {
@@ -105,6 +108,14 @@ func Cleanup(taskCtx context.Context, olderThan time.Duration) error {
 				}
 			}
 		}
+
+		if anyVersionDeleted {
+			if pcr.Type == packages_model.TypeDebian {
+				if err := debian_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil {
+					return fmt.Errorf("CleanupRule [%d]: debian.BuildAllRepositoryFiles failed: %w", pcr.ID, err)
+				}
+			}
+		}
 		return nil
 	})
 	if err != nil {
diff --git a/services/packages/debian/repository.go b/services/packages/debian/repository.go
new file mode 100644
index 0000000000..eac878256a
--- /dev/null
+++ b/services/packages/debian/repository.go
@@ -0,0 +1,443 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package debian
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+	"time"
+
+	"code.gitea.io/gitea/models/db"
+	packages_model "code.gitea.io/gitea/models/packages"
+	debian_model "code.gitea.io/gitea/models/packages/debian"
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/log"
+	packages_module "code.gitea.io/gitea/modules/packages"
+	debian_module "code.gitea.io/gitea/modules/packages/debian"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/util"
+	packages_service "code.gitea.io/gitea/services/packages"
+
+	"github.com/keybase/go-crypto/openpgp"
+	"github.com/keybase/go-crypto/openpgp/armor"
+	"github.com/keybase/go-crypto/openpgp/clearsign"
+	"github.com/keybase/go-crypto/openpgp/packet"
+	"github.com/ulikunitz/xz"
+)
+
+// GetOrCreateRepositoryVersion gets or creates the internal repository package
+// The Debian registry needs multiple index files which are stored in this package.
+func GetOrCreateRepositoryVersion(ownerID int64) (*packages_model.PackageVersion, error) {
+	var repositoryVersion *packages_model.PackageVersion
+
+	return repositoryVersion, db.WithTx(db.DefaultContext, func(ctx context.Context) error {
+		p := &packages_model.Package{
+			OwnerID:    ownerID,
+			Type:       packages_model.TypeDebian,
+			Name:       debian_module.RepositoryPackage,
+			LowerName:  debian_module.RepositoryPackage,
+			IsInternal: true,
+		}
+		var err error
+		if p, err = packages_model.TryInsertPackage(ctx, p); err != nil {
+			if err != packages_model.ErrDuplicatePackage {
+				log.Error("Error inserting package: %v", err)
+				return err
+			}
+		}
+
+		pv := &packages_model.PackageVersion{
+			PackageID:    p.ID,
+			CreatorID:    ownerID,
+			Version:      debian_module.RepositoryVersion,
+			LowerVersion: debian_module.RepositoryVersion,
+			IsInternal:   true,
+			MetadataJSON: "null",
+		}
+		if pv, err = packages_model.GetOrInsertVersion(ctx, pv); err != nil {
+			if err != packages_model.ErrDuplicatePackageVersion {
+				log.Error("Error inserting package version: %v", err)
+				return err
+			}
+		}
+
+		repositoryVersion = pv
+
+		return nil
+	})
+}
+
+// GetOrCreateKeyPair gets or creates the PGP keys used to sign repository files
+func GetOrCreateKeyPair(ownerID int64) (string, string, error) {
+	priv, err := user_model.GetSetting(ownerID, debian_module.SettingKeyPrivate)
+	if err != nil && !errors.Is(err, util.ErrNotExist) {
+		return "", "", err
+	}
+
+	pub, err := user_model.GetSetting(ownerID, debian_module.SettingKeyPublic)
+	if err != nil && !errors.Is(err, util.ErrNotExist) {
+		return "", "", err
+	}
+
+	if priv == "" || pub == "" {
+		priv, pub, err = generateKeypair()
+		if err != nil {
+			return "", "", err
+		}
+
+		if err := user_model.SetUserSetting(ownerID, debian_module.SettingKeyPrivate, priv); err != nil {
+			return "", "", err
+		}
+
+		if err := user_model.SetUserSetting(ownerID, debian_module.SettingKeyPublic, pub); err != nil {
+			return "", "", err
+		}
+	}
+
+	return priv, pub, nil
+}
+
+func generateKeypair() (string, string, error) {
+	e, err := openpgp.NewEntity(setting.AppName, "Debian Registry", "", nil)
+	if err != nil {
+		return "", "", err
+	}
+
+	var priv strings.Builder
+	var pub strings.Builder
+
+	w, err := armor.Encode(&priv, openpgp.PrivateKeyType, nil)
+	if err != nil {
+		return "", "", err
+	}
+	if err := e.SerializePrivate(w, nil); err != nil {
+		return "", "", err
+	}
+	w.Close()
+
+	w, err = armor.Encode(&pub, openpgp.PublicKeyType, nil)
+	if err != nil {
+		return "", "", err
+	}
+	if err := e.Serialize(w); err != nil {
+		return "", "", err
+	}
+	w.Close()
+
+	return priv.String(), pub.String(), nil
+}
+
+// BuildAllRepositoryFiles (re)builds all repository files for every available distributions, components and architectures
+func BuildAllRepositoryFiles(ctx context.Context, ownerID int64) error {
+	pv, err := GetOrCreateRepositoryVersion(ownerID)
+	if err != nil {
+		return err
+	}
+
+	// 1. Delete all existing repository files
+	pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID)
+	if err != nil {
+		return err
+	}
+
+	for _, pf := range pfs {
+		if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil {
+			return err
+		}
+		if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil {
+			return err
+		}
+	}
+
+	// 2. (Re)Build repository files for existing packages
+	distributions, err := debian_model.GetDistributions(ctx, ownerID)
+	if err != nil {
+		return err
+	}
+	for _, distribution := range distributions {
+		components, err := debian_model.GetComponents(ctx, ownerID, distribution)
+		if err != nil {
+			return err
+		}
+		architectures, err := debian_model.GetArchitectures(ctx, ownerID, distribution)
+		if err != nil {
+			return err
+		}
+
+		for _, component := range components {
+			for _, architecture := range architectures {
+				if err := buildRepositoryFiles(ctx, ownerID, pv, distribution, component, architecture); err != nil {
+					return fmt.Errorf("failed to build repository files [%s/%s/%s]: %w", distribution, component, architecture, err)
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+// BuildSpecificRepositoryFiles builds index files for the repository
+func BuildSpecificRepositoryFiles(ctx context.Context, ownerID int64, distribution, component, architecture string) error {
+	pv, err := GetOrCreateRepositoryVersion(ownerID)
+	if err != nil {
+		return err
+	}
+
+	return buildRepositoryFiles(ctx, ownerID, pv, distribution, component, architecture)
+}
+
+func buildRepositoryFiles(ctx context.Context, ownerID int64, repoVersion *packages_model.PackageVersion, distribution, component, architecture string) error {
+	if err := buildPackagesIndices(ctx, ownerID, repoVersion, distribution, component, architecture); err != nil {
+		return err
+	}
+
+	return buildReleaseFiles(ctx, ownerID, repoVersion, distribution)
+}
+
+// https://wiki.debian.org/DebianRepository/Format#A.22Packages.22_Indices
+func buildPackagesIndices(ctx context.Context, ownerID int64, repoVersion *packages_model.PackageVersion, distribution, component, architecture string) error {
+	pfds, err := debian_model.SearchLatestPackages(ctx, &debian_model.PackageSearchOptions{
+		OwnerID:      ownerID,
+		Distribution: distribution,
+		Component:    component,
+		Architecture: architecture,
+	})
+	if err != nil {
+		return err
+	}
+
+	// Delete the package indices if there are no packages
+	if len(pfds) == 0 {
+		key := fmt.Sprintf("%s|%s|%s", distribution, component, architecture)
+		for _, filename := range []string{"Packages", "Packages.gz", "Packages.xz"} {
+			pf, err := packages_model.GetFileForVersionByName(ctx, repoVersion.ID, filename, key)
+			if err != nil && !errors.Is(err, util.ErrNotExist) {
+				return err
+			}
+
+			if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil {
+				return err
+			}
+			if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}
+
+	packagesContent, _ := packages_module.NewHashedBuffer()
+
+	packagesGzipContent, _ := packages_module.NewHashedBuffer()
+	gzw := gzip.NewWriter(packagesGzipContent)
+
+	packagesXzContent, _ := packages_module.NewHashedBuffer()
+	xzw, _ := xz.NewWriter(packagesXzContent)
+
+	w := io.MultiWriter(packagesContent, gzw, xzw)
+
+	addSeperator := false
+	for _, pfd := range pfds {
+		if addSeperator {
+			fmt.Fprintln(w)
+		}
+		addSeperator = true
+
+		fmt.Fprint(w, pfd.Properties.GetByName(debian_module.PropertyControl))
+
+		fmt.Fprintf(w, "Filename: pool/%s/%s/%s\n", distribution, component, pfd.File.Name)
+		fmt.Fprintf(w, "Size: %d\n", pfd.Blob.Size)
+		fmt.Fprintf(w, "MD5sum: %s\n", pfd.Blob.HashMD5)
+		fmt.Fprintf(w, "SHA1: %s\n", pfd.Blob.HashSHA1)
+		fmt.Fprintf(w, "SHA256: %s\n", pfd.Blob.HashSHA256)
+		fmt.Fprintf(w, "SHA512: %s\n", pfd.Blob.HashSHA512)
+	}
+
+	gzw.Close()
+	xzw.Close()
+
+	for _, file := range []struct {
+		Name string
+		Data packages_module.HashedSizeReader
+	}{
+		{"Packages", packagesContent},
+		{"Packages.gz", packagesGzipContent},
+		{"Packages.xz", packagesXzContent},
+	} {
+		_, err = packages_service.AddFileToPackageVersionInternal(
+			repoVersion,
+			&packages_service.PackageFileCreationInfo{
+				PackageFileInfo: packages_service.PackageFileInfo{
+					Filename:     file.Name,
+					CompositeKey: fmt.Sprintf("%s|%s|%s", distribution, component, architecture),
+				},
+				Creator:           user_model.NewGhostUser(),
+				Data:              file.Data,
+				IsLead:            false,
+				OverwriteExisting: true,
+				Properties: map[string]string{
+					debian_module.PropertyRepositoryIncludeInRelease: "",
+					debian_module.PropertyDistribution:               distribution,
+					debian_module.PropertyComponent:                  component,
+					debian_module.PropertyArchitecture:               architecture,
+				},
+			},
+		)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// https://wiki.debian.org/DebianRepository/Format#A.22Release.22_files
+func buildReleaseFiles(ctx context.Context, ownerID int64, repoVersion *packages_model.PackageVersion, distribution string) error {
+	pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{
+		VersionID: repoVersion.ID,
+		Properties: map[string]string{
+			debian_module.PropertyRepositoryIncludeInRelease: "",
+			debian_module.PropertyDistribution:               distribution,
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	// Delete the release files if there are no packages
+	if len(pfs) == 0 {
+		for _, filename := range []string{"Release", "Release.gpg", "InRelease"} {
+			pf, err := packages_model.GetFileForVersionByName(ctx, repoVersion.ID, filename, distribution)
+			if err != nil && !errors.Is(err, util.ErrNotExist) {
+				return err
+			}
+
+			if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil {
+				return err
+			}
+			if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}
+
+	components, err := debian_model.GetComponents(ctx, ownerID, distribution)
+	if err != nil {
+		return err
+	}
+
+	sort.Strings(components)
+
+	architectures, err := debian_model.GetArchitectures(ctx, ownerID, distribution)
+	if err != nil {
+		return err
+	}
+
+	sort.Strings(architectures)
+
+	priv, _, err := GetOrCreateKeyPair(ownerID)
+	if err != nil {
+		return err
+	}
+
+	block, err := armor.Decode(strings.NewReader(priv))
+	if err != nil {
+		return err
+	}
+
+	e, err := openpgp.ReadEntity(packet.NewReader(block.Body))
+	if err != nil {
+		return err
+	}
+
+	inReleaseContent, _ := packages_module.NewHashedBuffer()
+	sw, err := clearsign.Encode(inReleaseContent, e.PrivateKey, nil)
+	if err != nil {
+		return err
+	}
+
+	var buf bytes.Buffer
+
+	w := io.MultiWriter(sw, &buf)
+
+	fmt.Fprintf(w, "Origin: %s\n", setting.AppName)
+	fmt.Fprintf(w, "Label: %s\n", setting.AppName)
+	fmt.Fprintf(w, "Suite: %s\n", distribution)
+	fmt.Fprintf(w, "Codename: %s\n", distribution)
+	fmt.Fprintf(w, "Components: %s\n", strings.Join(components, " "))
+	fmt.Fprintf(w, "Architectures: %s\n", strings.Join(architectures, " "))
+	fmt.Fprintf(w, "Date: %s\n", time.Now().UTC().Format(time.RFC1123))
+	fmt.Fprint(w, "Acquire-By-Hash: yes")
+
+	pfds, err := packages_model.GetPackageFileDescriptors(ctx, pfs)
+	if err != nil {
+		return err
+	}
+
+	var md5, sha1, sha256, sha512 strings.Builder
+	for _, pfd := range pfds {
+		path := fmt.Sprintf("%s/binary-%s/%s", pfd.Properties.GetByName(debian_module.PropertyComponent), pfd.Properties.GetByName(debian_module.PropertyArchitecture), pfd.File.Name)
+		fmt.Fprintf(&md5, " %s %d %s\n", pfd.Blob.HashMD5, pfd.Blob.Size, path)
+		fmt.Fprintf(&sha1, " %s %d %s\n", pfd.Blob.HashSHA1, pfd.Blob.Size, path)
+		fmt.Fprintf(&sha256, " %s %d %s\n", pfd.Blob.HashSHA256, pfd.Blob.Size, path)
+		fmt.Fprintf(&sha512, " %s %d %s\n", pfd.Blob.HashSHA512, pfd.Blob.Size, path)
+	}
+
+	fmt.Fprintln(w, "MD5Sum:")
+	fmt.Fprint(w, md5.String())
+	fmt.Fprintln(w, "SHA1:")
+	fmt.Fprint(w, sha1.String())
+	fmt.Fprintln(w, "SHA256:")
+	fmt.Fprint(w, sha256.String())
+	fmt.Fprintln(w, "SHA512:")
+	fmt.Fprint(w, sha512.String())
+
+	sw.Close()
+
+	releaseGpgContent, _ := packages_module.NewHashedBuffer()
+	if err := openpgp.ArmoredDetachSign(releaseGpgContent, e, bytes.NewReader(buf.Bytes()), nil); err != nil {
+		return err
+	}
+
+	releaseContent, _ := packages_module.CreateHashedBufferFromReader(&buf)
+
+	for _, file := range []struct {
+		Name string
+		Data packages_module.HashedSizeReader
+	}{
+		{"Release", releaseContent},
+		{"Release.gpg", releaseGpgContent},
+		{"InRelease", inReleaseContent},
+	} {
+		_, err = packages_service.AddFileToPackageVersionInternal(
+			repoVersion,
+			&packages_service.PackageFileCreationInfo{
+				PackageFileInfo: packages_service.PackageFileInfo{
+					Filename:     file.Name,
+					CompositeKey: distribution,
+				},
+				Creator:           user_model.NewGhostUser(),
+				Data:              file.Data,
+				IsLead:            false,
+				OverwriteExisting: true,
+				Properties: map[string]string{
+					debian_module.PropertyDistribution: distribution,
+				},
+			},
+		)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
diff --git a/services/packages/packages.go b/services/packages/packages.go
index dd5c63470b..735e52c854 100644
--- a/services/packages/packages.go
+++ b/services/packages/packages.go
@@ -187,19 +187,33 @@ func createPackageAndVersion(ctx context.Context, pvci *PackageCreationInfo, all
 }
 
 // AddFileToExistingPackage adds a file to an existing package. If the package does not exist, ErrPackageNotExist is returned
-func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageVersion, *packages_model.PackageFile, error) {
+func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, error) {
+	return addFileToPackageWrapper(func(ctx context.Context) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) {
+		pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version)
+		if err != nil {
+			return nil, nil, false, err
+		}
+
+		return addFileToPackageVersion(ctx, pv, pvi, pfci)
+	})
+}
+
+// AddFileToPackageVersionInternal adds a file to the package
+// This method skips quota checks and should only be used for system-managed packages.
+func AddFileToPackageVersionInternal(pv *packages_model.PackageVersion, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, error) {
+	return addFileToPackageWrapper(func(ctx context.Context) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) {
+		return addFileToPackageVersionUnchecked(ctx, pv, pfci)
+	})
+}
+
+func addFileToPackageWrapper(fn func(ctx context.Context) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error)) (*packages_model.PackageFile, error) {
 	ctx, committer, err := db.TxContext(db.DefaultContext)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 	defer committer.Close()
 
-	pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pvi, pfci)
+	pf, pb, blobCreated, err := fn(ctx)
 	removeBlob := false
 	defer func() {
 		if removeBlob {
@@ -211,15 +225,15 @@ func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) (
 	}()
 	if err != nil {
 		removeBlob = blobCreated
-		return nil, nil, err
+		return nil, err
 	}
 
 	if err := committer.Commit(); err != nil {
 		removeBlob = blobCreated
-		return nil, nil, err
+		return nil, err
 	}
 
-	return pv, pf, nil
+	return pf, nil
 }
 
 // NewPackageBlob creates a package blob instance
@@ -236,12 +250,16 @@ func NewPackageBlob(hsr packages_module.HashedSizeReader) *packages_model.Packag
 }
 
 func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pvi *PackageInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) {
-	log.Trace("Adding package file: %v, %s", pv.ID, pfci.Filename)
-
 	if err := CheckSizeQuotaExceeded(ctx, pfci.Creator, pvi.Owner, pvi.PackageType, pfci.Data.Size()); err != nil {
 		return nil, nil, false, err
 	}
 
+	return addFileToPackageVersionUnchecked(ctx, pv, pfci)
+}
+
+func addFileToPackageVersionUnchecked(ctx context.Context, pv *packages_model.PackageVersion, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) {
+	log.Trace("Adding package file: %v, %s", pv.ID, pfci.Filename)
+
 	pb, exists, err := packages_model.GetOrInsertBlob(ctx, NewPackageBlob(pfci.Data))
 	if err != nil {
 		log.Error("Error inserting package blob: %v", err)
@@ -345,6 +363,8 @@ func CheckSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, p
 		typeSpecificSize = setting.Packages.LimitSizeConda
 	case packages_model.TypeContainer:
 		typeSpecificSize = setting.Packages.LimitSizeContainer
+	case packages_model.TypeDebian:
+		typeSpecificSize = setting.Packages.LimitSizeDebian
 	case packages_model.TypeGeneric:
 		typeSpecificSize = setting.Packages.LimitSizeGeneric
 	case packages_model.TypeHelm: