Add package registry cleanup rules (#21658)

Fixes #20514
Fixes #20766
Fixes #20631

This PR adds Cleanup Rules for the package registry. This allows to
delete unneeded packages automatically. Cleanup rules can be set up from
the user or org settings.
Please have a look at the documentation because I'm not a native english
speaker.

Rule Form

![grafik](https://user-images.githubusercontent.com/1666336/199330792-c13918a6-e196-4e71-9f53-18554515edca.png)

Rule List

![grafik](https://user-images.githubusercontent.com/1666336/199331261-5f6878e8-a80c-4985-800d-ebb3524b1a8d.png)

Rule Preview

![grafik](https://user-images.githubusercontent.com/1666336/199330917-c95e4017-cf64-4142-a3e4-af18c4f127c3.png)

Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>

1 files changed, 69 insertions, 2 deletions

diff --git a/services/packages/packages.go b/services/packages/packages.go
index 76fdd02bf2..7343ffc530 100644
--- a/services/packages/packages.go
+++ b/services/packages/packages.go
@@ -443,13 +443,80 @@ func DeletePackageFile(ctx context.Context, pf *packages_model.PackageFile) erro
 }
 
 // Cleanup removes expired package data
-func Cleanup(unused context.Context, olderThan time.Duration) error {
-	ctx, committer, err := db.TxContext(db.DefaultContext)
+func Cleanup(taskCtx context.Context, olderThan time.Duration) error {
+	ctx, committer, err := db.TxContext(taskCtx)
 	if err != nil {
 		return err
 	}
 	defer committer.Close()
 
+	err = packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error {
+		select {
+		case <-taskCtx.Done():
+			return db.ErrCancelledf("While processing package cleanup rules")
+		default:
+		}
+
+		if err := pcr.CompiledPattern(); err != nil {
+			return fmt.Errorf("CleanupRule [%d]: CompilePattern failed: %w", pcr.ID, err)
+		}
+
+		olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays)
+
+		packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type)
+		if err != nil {
+			return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err)
+		}
+
+		for _, p := range packages {
+			pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
+				PackageID:  p.ID,
+				IsInternal: util.OptionalBoolFalse,
+				Sort:       packages_model.SortCreatedDesc,
+				Paginator:  db.NewAbsoluteListOptions(pcr.KeepCount, 200),
+			})
+			if err != nil {
+				return fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err)
+			}
+			for _, pv := range pvs {
+				if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil {
+					return fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err)
+				} else if skip {
+					log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version)
+					continue
+				}
+
+				toMatch := pv.LowerVersion
+				if pcr.MatchFullName {
+					toMatch = p.LowerName + "/" + pv.LowerVersion
+				}
+
+				if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) {
+					log.Debug("Rule[%d]: keep '%s/%s' (keep pattern)", pcr.ID, p.Name, pv.Version)
+					continue
+				}
+				if pv.CreatedUnix.AsLocalTime().After(olderThan) {
+					log.Debug("Rule[%d]: keep '%s/%s' (remove days)", pcr.ID, p.Name, pv.Version)
+					continue
+				}
+				if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) {
+					log.Debug("Rule[%d]: keep '%s/%s' (remove pattern)", pcr.ID, p.Name, pv.Version)
+					continue
+				}
+
+				log.Debug("Rule[%d]: remove '%s/%s'", pcr.ID, p.Name, pv.Version)
+
+				if err := DeletePackageVersionAndReferences(ctx, pv); err != nil {
+					return fmt.Errorf("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %w", pcr.ID, err)
+				}
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
 	if err := container_service.Cleanup(ctx, olderThan); err != nil {
 		return err
 	}