aboutsummaryrefslogtreecommitdiffstats
path: root/services/packages
diff options
context:
space:
mode:
Diffstat (limited to 'services/packages')
-rw-r--r--services/packages/arch/vercmp.go9
-rw-r--r--services/packages/cargo/index.go2
-rw-r--r--services/packages/cleanup/cleanup.go211
-rw-r--r--services/packages/container/cleanup.go4
-rw-r--r--services/packages/container/common.go31
-rw-r--r--services/packages/packages.go30
-rw-r--r--services/packages/rpm/repository.go4
7 files changed, 165 insertions, 126 deletions
diff --git a/services/packages/arch/vercmp.go b/services/packages/arch/vercmp.go
index 0d33dda0f1..d44aa530f0 100644
--- a/services/packages/arch/vercmp.go
+++ b/services/packages/arch/vercmp.go
@@ -34,13 +34,8 @@ func parseEVR(evr string) (epoch, version, release string) {
func compareSegments(a, b []string) int {
lenA, lenB := len(a), len(b)
- var l int
- if lenA > lenB {
- l = lenB
- } else {
- l = lenA
- }
- for i := 0; i < l; i++ {
+ l := min(lenA, lenB)
+ for i := range l {
if r := compare(a[i], b[i]); r != 0 {
return r
}
diff --git a/services/packages/cargo/index.go b/services/packages/cargo/index.go
index e8a2f189c8..605335d0f1 100644
--- a/services/packages/cargo/index.go
+++ b/services/packages/cargo/index.go
@@ -310,7 +310,7 @@ func alterRepositoryContent(ctx context.Context, doer *user_model.User, repo *re
}
func writeObjectToIndex(ctx context.Context, t *files_service.TemporaryUploadRepository, path string, r io.Reader) error {
- hash, err := t.HashObject(ctx, r)
+ hash, err := t.HashObjectAndWrite(ctx, r)
if err != nil {
return err
}
diff --git a/services/packages/cleanup/cleanup.go b/services/packages/cleanup/cleanup.go
index b7ba2b6ac4..959babe7cd 100644
--- a/services/packages/cleanup/cleanup.go
+++ b/services/packages/cleanup/cleanup.go
@@ -32,127 +32,136 @@ func CleanupTask(ctx context.Context, olderThan time.Duration) error {
return CleanupExpiredData(ctx, olderThan)
}
-func ExecuteCleanupRules(outerCtx context.Context) error {
- ctx, committer, err := db.TxContext(outerCtx)
+func executeCleanupOneRulePackage(ctx context.Context, pcr *packages_model.PackageCleanupRule, p *packages_model.Package) (versionDeleted bool, err error) {
+ olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays)
+ pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
+ PackageID: p.ID,
+ IsInternal: optional.Some(false),
+ Sort: packages_model.SortCreatedDesc,
+ })
if err != nil {
- return err
+ return false, fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err)
}
- defer committer.Close()
-
- err = packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error {
- select {
- case <-outerCtx.Done():
- return db.ErrCancelledf("While processing package cleanup rules")
- default:
+ if pcr.KeepCount > 0 {
+ if pcr.KeepCount < len(pvs) {
+ pvs = pvs[pcr.KeepCount:]
+ } else {
+ pvs = nil
}
-
- if err := pcr.CompiledPattern(); err != nil {
- return fmt.Errorf("CleanupRule [%d]: CompilePattern failed: %w", pcr.ID, err)
+ }
+ for _, pv := range pvs {
+ if pcr.Type == packages_model.TypeContainer {
+ if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil {
+ return false, fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err)
+ } else if skip {
+ log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version)
+ continue
+ }
}
-
- olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays)
-
- packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type)
- if err != nil {
- return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err)
+ toMatch := pv.LowerVersion
+ if pcr.MatchFullName {
+ toMatch = p.LowerName + "/" + pv.LowerVersion
+ }
+ if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) {
+ log.Debug("Rule[%d]: keep '%s/%s' (keep pattern)", pcr.ID, p.Name, pv.Version)
+ continue
+ }
+ if pv.CreatedUnix.AsLocalTime().After(olderThan) {
+ log.Debug("Rule[%d]: keep '%s/%s' (remove days) %v", pcr.ID, p.Name, pv.Version, pv.CreatedUnix.FormatDate())
+ continue
}
+ if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) {
+ log.Debug("Rule[%d]: keep '%s/%s' (remove pattern)", pcr.ID, p.Name, pv.Version)
+ continue
+ }
+ log.Debug("Rule[%d]: remove '%s/%s'", pcr.ID, p.Name, pv.Version)
+ if err := packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil {
+ log.Error("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %v", pcr.ID, err)
+ continue
+ }
+ versionDeleted = true
+ }
+ return versionDeleted, nil
+}
- anyVersionDeleted := false
- for _, p := range packages {
- pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
- PackageID: p.ID,
- IsInternal: optional.Some(false),
- Sort: packages_model.SortCreatedDesc,
- Paginator: db.NewAbsoluteListOptions(pcr.KeepCount, 200),
- })
- if err != nil {
- return fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err)
- }
- versionDeleted := false
- for _, pv := range pvs {
- if pcr.Type == packages_model.TypeContainer {
- if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil {
- return fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err)
- } else if skip {
- log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version)
- continue
- }
- }
+func executeCleanupOneRule(ctx context.Context, pcr *packages_model.PackageCleanupRule) error {
+ if err := pcr.CompiledPattern(); err != nil {
+ return fmt.Errorf("CleanupRule [%d]: CompilePattern failed: %w", pcr.ID, err)
+ }
- toMatch := pv.LowerVersion
- if pcr.MatchFullName {
- toMatch = p.LowerName + "/" + pv.LowerVersion
- }
+ packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type)
+ if err != nil {
+ return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err)
+ }
- if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) {
- log.Debug("Rule[%d]: keep '%s/%s' (keep pattern)", pcr.ID, p.Name, pv.Version)
- continue
- }
- if pv.CreatedUnix.AsLocalTime().After(olderThan) {
- log.Debug("Rule[%d]: keep '%s/%s' (remove days)", pcr.ID, p.Name, pv.Version)
- continue
- }
- if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) {
- log.Debug("Rule[%d]: keep '%s/%s' (remove pattern)", pcr.ID, p.Name, pv.Version)
- continue
+ anyVersionDeleted := false
+ for _, p := range packages {
+ versionDeleted := false
+ err = db.WithTx(ctx, func(ctx context.Context) (err error) {
+ versionDeleted, err = executeCleanupOneRulePackage(ctx, pcr, p)
+ return err
+ })
+ if err != nil {
+ log.Error("CleanupRule [%d]: executeCleanupOneRulePackage(%d) failed: %v", pcr.ID, p.ID, err)
+ continue
+ }
+ anyVersionDeleted = anyVersionDeleted || versionDeleted
+ if versionDeleted {
+ if pcr.Type == packages_model.TypeCargo {
+ owner, err := user_model.GetUserByID(ctx, pcr.OwnerID)
+ if err != nil {
+ return fmt.Errorf("GetUserByID failed: %w", err)
}
-
- log.Debug("Rule[%d]: remove '%s/%s'", pcr.ID, p.Name, pv.Version)
-
- if err := packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil {
- return fmt.Errorf("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %w", pcr.ID, err)
+ if err := cargo_service.UpdatePackageIndexIfExists(ctx, owner, owner, p.ID); err != nil {
+ return fmt.Errorf("CleanupRule [%d]: cargo.UpdatePackageIndexIfExists failed: %w", pcr.ID, err)
}
+ }
+ }
+ }
- versionDeleted = true
- anyVersionDeleted = true
+ if anyVersionDeleted {
+ switch pcr.Type {
+ case packages_model.TypeDebian:
+ if err := debian_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil {
+ return fmt.Errorf("CleanupRule [%d]: debian.BuildAllRepositoryFiles failed: %w", pcr.ID, err)
+ }
+ case packages_model.TypeAlpine:
+ if err := alpine_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil {
+ return fmt.Errorf("CleanupRule [%d]: alpine.BuildAllRepositoryFiles failed: %w", pcr.ID, err)
}
+ case packages_model.TypeRpm:
+ if err := rpm_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil {
+ return fmt.Errorf("CleanupRule [%d]: rpm.BuildAllRepositoryFiles failed: %w", pcr.ID, err)
+ }
+ case packages_model.TypeArch:
+ release, err := arch_service.AquireRegistryLock(ctx, pcr.OwnerID)
+ if err != nil {
+ return err
+ }
+ defer release()
- if versionDeleted {
- if pcr.Type == packages_model.TypeCargo {
- owner, err := user_model.GetUserByID(ctx, pcr.OwnerID)
- if err != nil {
- return fmt.Errorf("GetUserByID failed: %w", err)
- }
- if err := cargo_service.UpdatePackageIndexIfExists(ctx, owner, owner, p.ID); err != nil {
- return fmt.Errorf("CleanupRule [%d]: cargo.UpdatePackageIndexIfExists failed: %w", pcr.ID, err)
- }
- }
+ if err := arch_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil {
+ return fmt.Errorf("CleanupRule [%d]: arch.BuildAllRepositoryFiles failed: %w", pcr.ID, err)
}
}
+ }
+ return nil
+}
- if anyVersionDeleted {
- switch pcr.Type {
- case packages_model.TypeDebian:
- if err := debian_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil {
- return fmt.Errorf("CleanupRule [%d]: debian.BuildAllRepositoryFiles failed: %w", pcr.ID, err)
- }
- case packages_model.TypeAlpine:
- if err := alpine_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil {
- return fmt.Errorf("CleanupRule [%d]: alpine.BuildAllRepositoryFiles failed: %w", pcr.ID, err)
- }
- case packages_model.TypeRpm:
- if err := rpm_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil {
- return fmt.Errorf("CleanupRule [%d]: rpm.BuildAllRepositoryFiles failed: %w", pcr.ID, err)
- }
- case packages_model.TypeArch:
- release, err := arch_service.AquireRegistryLock(ctx, pcr.OwnerID)
- if err != nil {
- return err
- }
- defer release()
+func ExecuteCleanupRules(ctx context.Context) error {
+ return packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error {
+ select {
+ case <-ctx.Done():
+ return db.ErrCancelledf("While processing package cleanup rules")
+ default:
+ }
- if err := arch_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil {
- return fmt.Errorf("CleanupRule [%d]: arch.BuildAllRepositoryFiles failed: %w", pcr.ID, err)
- }
- }
+ err := executeCleanupOneRule(ctx, pcr)
+ if err != nil {
+ log.Error("CleanupRule [%d]: executeCleanupOneRule failed: %v", pcr.ID, err)
}
return nil
})
- if err != nil {
- return err
- }
-
- return committer.Commit()
}
func CleanupExpiredData(outerCtx context.Context, olderThan time.Duration) error {
diff --git a/services/packages/container/cleanup.go b/services/packages/container/cleanup.go
index 3f5f43bbc0..263562a396 100644
--- a/services/packages/container/cleanup.go
+++ b/services/packages/container/cleanup.go
@@ -13,7 +13,7 @@ import (
container_module "code.gitea.io/gitea/modules/packages/container"
packages_service "code.gitea.io/gitea/services/packages"
- digest "github.com/opencontainers/go-digest"
+ "github.com/opencontainers/go-digest"
)
// Cleanup removes expired container data
@@ -57,7 +57,7 @@ func cleanupExpiredUploadedBlobs(ctx context.Context, olderThan time.Duration) e
Type: packages_model.TypeContainer,
Version: packages_model.SearchValue{
ExactMatch: true,
- Value: container_model.UploadVersion,
+ Value: container_module.UploadVersion,
},
IsInternal: optional.Some(true),
HasFiles: optional.Some(false),
diff --git a/services/packages/container/common.go b/services/packages/container/common.go
index 5a14ed5b7a..02cbff2286 100644
--- a/services/packages/container/common.go
+++ b/services/packages/container/common.go
@@ -5,11 +5,17 @@ package container
import (
"context"
+ "io"
"strings"
packages_model "code.gitea.io/gitea/models/packages"
+ container_service "code.gitea.io/gitea/models/packages/container"
user_model "code.gitea.io/gitea/models/user"
+ "code.gitea.io/gitea/modules/json"
+ "code.gitea.io/gitea/modules/packages"
container_module "code.gitea.io/gitea/modules/packages/container"
+
+ "github.com/opencontainers/image-spec/specs-go/v1"
)
// UpdateRepositoryNames updates the repository name property for all packages of the specific owner
@@ -22,7 +28,7 @@ func UpdateRepositoryNames(ctx context.Context, owner *user_model.User, newOwner
newOwnerName = strings.ToLower(newOwnerName)
for _, p := range ps {
- if err := packages_model.DeletePropertyByName(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository); err != nil {
+ if err := packages_model.DeletePropertiesByName(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository); err != nil {
return err
}
@@ -33,3 +39,26 @@ func UpdateRepositoryNames(ctx context.Context, owner *user_model.User, newOwner
return nil
}
+
+func ParseManifestMetadata(ctx context.Context, rd io.Reader, ownerID int64, imageName string) (*v1.Manifest, *packages_model.PackageFileDescriptor, *container_module.Metadata, error) {
+ var manifest v1.Manifest
+ if err := json.NewDecoder(rd).Decode(&manifest); err != nil {
+ return nil, nil, nil, err
+ }
+ configDescriptor, err := container_service.GetContainerBlob(ctx, &container_service.BlobSearchOptions{
+ OwnerID: ownerID,
+ Image: imageName,
+ Digest: manifest.Config.Digest.String(),
+ })
+ if err != nil {
+ return nil, nil, nil, err
+ }
+
+ configReader, err := packages.NewContentStore().OpenBlob(packages.BlobHash256Key(configDescriptor.Blob.HashSHA256))
+ if err != nil {
+ return nil, nil, nil, err
+ }
+ defer configReader.Close()
+ metadata, err := container_module.ParseImageConfig(manifest.Config.MediaType, configReader)
+ return &manifest, configDescriptor, metadata, err
+}
diff --git a/services/packages/packages.go b/services/packages/packages.go
index bd1d460fd3..517334cbc7 100644
--- a/services/packages/packages.go
+++ b/services/packages/packages.go
@@ -563,8 +563,8 @@ func DeletePackageFile(ctx context.Context, pf *packages_model.PackageFile) erro
return packages_model.DeleteFileByID(ctx, pf.ID)
}
-// GetFileStreamByPackageNameAndVersion returns the content of the specific package file
-func GetFileStreamByPackageNameAndVersion(ctx context.Context, pvi *PackageInfo, pfi *PackageFileInfo) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) {
+// OpenFileForDownloadByPackageNameAndVersion returns the content of the specific package file and increases the download counter.
+func OpenFileForDownloadByPackageNameAndVersion(ctx context.Context, pvi *PackageInfo, pfi *PackageFileInfo) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) {
log.Trace("Getting package file stream: %v, %v, %s, %s, %s, %s", pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, pfi.Filename, pfi.CompositeKey)
pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version)
@@ -576,32 +576,38 @@ func GetFileStreamByPackageNameAndVersion(ctx context.Context, pvi *PackageInfo,
return nil, nil, nil, err
}
- return GetFileStreamByPackageVersion(ctx, pv, pfi)
+ return OpenFileForDownloadByPackageVersion(ctx, pv, pfi)
}
-// GetFileStreamByPackageVersion returns the content of the specific package file
-func GetFileStreamByPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pfi *PackageFileInfo) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) {
+// OpenFileForDownloadByPackageVersion returns the content of the specific package file and increases the download counter.
+func OpenFileForDownloadByPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pfi *PackageFileInfo) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) {
pf, err := packages_model.GetFileForVersionByName(ctx, pv.ID, pfi.Filename, pfi.CompositeKey)
if err != nil {
return nil, nil, nil, err
}
- return GetPackageFileStream(ctx, pf)
+ return OpenFileForDownload(ctx, pf)
}
-// GetPackageFileStream returns the content of the specific package file
-func GetPackageFileStream(ctx context.Context, pf *packages_model.PackageFile) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) {
+// OpenFileForDownload returns the content of the specific package file and increases the download counter.
+func OpenFileForDownload(ctx context.Context, pf *packages_model.PackageFile) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) {
pb, err := packages_model.GetBlobByID(ctx, pf.BlobID)
if err != nil {
return nil, nil, nil, err
}
- return GetPackageBlobStream(ctx, pf, pb, nil)
+ return OpenBlobForDownload(ctx, pf, pb, nil)
}
-// GetPackageBlobStream returns the content of the specific package blob
+func OpenBlobStream(pb *packages_model.PackageBlob) (io.ReadSeekCloser, error) {
+ cs := packages_module.NewContentStore()
+ key := packages_module.BlobHash256Key(pb.HashSHA256)
+ return cs.OpenBlob(key)
+}
+
+// OpenBlobForDownload returns the content of the specific package blob and increases the download counter.
// If the storage supports direct serving and it's enabled, only the direct serving url is returned.
-func GetPackageBlobStream(ctx context.Context, pf *packages_model.PackageFile, pb *packages_model.PackageBlob, serveDirectReqParams url.Values) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) {
+func OpenBlobForDownload(ctx context.Context, pf *packages_model.PackageFile, pb *packages_model.PackageBlob, serveDirectReqParams url.Values) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) {
key := packages_module.BlobHash256Key(pb.HashSHA256)
cs := packages_module.NewContentStore()
@@ -617,7 +623,7 @@ func GetPackageBlobStream(ctx context.Context, pf *packages_model.PackageFile, p
}
}
if u == nil {
- s, err = cs.Get(key)
+ s, err = cs.OpenBlob(key)
}
if err == nil {
diff --git a/services/packages/rpm/repository.go b/services/packages/rpm/repository.go
index 5027021c52..fbbf8d7dad 100644
--- a/services/packages/rpm/repository.go
+++ b/services/packages/rpm/repository.go
@@ -470,7 +470,7 @@ func buildPrimary(ctx context.Context, pv *packages_model.PackageVersion, pfs []
}
// https://docs.pulpproject.org/en/2.19/plugins/pulp_rpm/tech-reference/rpm.html#filelists-xml
-func buildFilelists(ctx context.Context, pv *packages_model.PackageVersion, pfs []*packages_model.PackageFile, c packageCache, group string) (*repoData, error) { //nolint:dupl
+func buildFilelists(ctx context.Context, pv *packages_model.PackageVersion, pfs []*packages_model.PackageFile, c packageCache, group string) (*repoData, error) { //nolint:dupl // duplicates with buildOther
type Version struct {
Epoch string `xml:"epoch,attr"`
Version string `xml:"ver,attr"`
@@ -517,7 +517,7 @@ func buildFilelists(ctx context.Context, pv *packages_model.PackageVersion, pfs
}
// https://docs.pulpproject.org/en/2.19/plugins/pulp_rpm/tech-reference/rpm.html#other-xml
-func buildOther(ctx context.Context, pv *packages_model.PackageVersion, pfs []*packages_model.PackageFile, c packageCache, group string) (*repoData, error) { //nolint:dupl
+func buildOther(ctx context.Context, pv *packages_model.PackageVersion, pfs []*packages_model.PackageFile, c packageCache, group string) (*repoData, error) { //nolint:dupl // duplicates with buildFilelists
type Version struct {
Epoch string `xml:"epoch,attr"`
Version string `xml:"ver,attr"`
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-02 23:32:53 +0000