Implement archive cleanup (#885)

* Implement archive cleanup

Fixes #769

Signed-off-by: Andrew <write@imaginarycode.com>
* Make sure to close the directory file

* Resolve issues noted by @strk

* edit cheatsheet app.ini [ci skip]

* oops [ci skip]

diff --git a/conf/app.ini b/conf/app.ini
index 254b1bc549e065d283d6a877629c375818613e06..8338a1b93dec0df487d35d4584d57350d106f8ad 100644 (file)
--- a/conf/app.ini
+++ b/conf/app.ini
@@ -391,6 +391,13 @@ ARGS =
 RUN_AT_START = true
 SCHEDULE = @every 24h
 
+; Clean up old repository archives
+[cron.archive_cleanup]
+RUN_AT_START = true
+SCHEDULE = @every 24h
+; Archives created more than OLDER_THAN ago are subject to deletion
+OLDER_THAN = 24h
+
 [git]
 ; Disables highlight of added and removed changes
 DISABLE_DIFF_HIGHLIGHT = false

diff --git a/models/repo.go b/models/repo.go
index 9b1b868778bfaaf7672263f9bb2e355c515aa650..a7a36cc7d09300541805e206657520958d3c6f02 100644 (file)
--- a/models/repo.go
+++ b/models/repo.go
@@ -1837,6 +1837,60 @@ func DeleteRepositoryArchives() error {
                        })
 }
 
+// DeleteOldRepositoryArchives deletes old repository archives.
+func DeleteOldRepositoryArchives() {
+       if taskStatusTable.IsRunning(archiveCleanup) {
+               return
+       }
+       taskStatusTable.Start(archiveCleanup)
+       defer taskStatusTable.Stop(archiveCleanup)
+
+       log.Trace("Doing: ArchiveCleanup")
+
+       if err := x.Where("id > 0").Iterate(new(Repository), deleteOldRepositoryArchives); err != nil {
+               log.Error(4, "ArchiveClean: %v", err)
+       }
+}
+
+func deleteOldRepositoryArchives(idx int, bean interface{}) error {
+       repo := bean.(*Repository)
+       basePath := filepath.Join(repo.RepoPath(), "archives")
+
+       for _, ty := range []string{"zip", "targz"} {
+               path := filepath.Join(basePath, ty)
+               file, err := os.Open(path)
+               if err != nil {
+                       if !os.IsNotExist(err) {
+                               log.Warn("Unable to open directory %s: %v", path, err)
+                               return err
+                       }
+
+                       // If the directory doesn't exist, that's okay.
+                       continue
+               }
+
+               files, err := file.Readdir(0)
+               file.Close()
+               if err != nil {
+                       log.Warn("Unable to read directory %s: %v", path, err)
+                       return err
+               }
+
+               minimumOldestTime := time.Now().Add(-setting.Cron.ArchiveCleanup.OlderThan)
+               for _, info := range files {
+                       if info.ModTime().Before(minimumOldestTime) && !info.IsDir() {
+                               toDelete := filepath.Join(path, info.Name())
+                               // This is a best-effort purge, so we do not check error codes to confirm removal.
+                               if err = os.Remove(toDelete); err != nil {
+                                       log.Trace("Unable to delete %s, but proceeding: %v", toDelete, err)
+                               }
+                       }
+               }
+       }
+
+       return nil
+}
+
 func gatherMissingRepoRecords() ([]*Repository, error) {
        repos := make([]*Repository, 0, 10)
        if err := x.
@@ -1915,9 +1969,10 @@ func RewriteRepositoryUpdateHook() error {
 var taskStatusTable = sync.NewStatusTable()
 
 const (
-       mirrorUpdate = "mirror_update"
-       gitFsck      = "git_fsck"
-       checkRepos   = "check_repos"
+       mirrorUpdate   = "mirror_update"
+       gitFsck        = "git_fsck"
+       checkRepos     = "check_repos"
+       archiveCleanup = "archive_cleanup"
 )
 
 // GitFsck calls 'git fsck' to check repository health.

diff --git a/modules/cron/cron.go b/modules/cron/cron.go
index e1110d787b300d5e99356d0ccbf28dafd74df017..785bf44ada075b8974a98414408edcb2428cd0d3 100644 (file)
--- a/modules/cron/cron.go
+++ b/modules/cron/cron.go
@@ -55,6 +55,17 @@ func NewContext() {
                        go models.CheckRepoStats()
                }
        }
+       if setting.Cron.ArchiveCleanup.Enabled {
+               entry, err = c.AddFunc("Clean up old repository archives", setting.Cron.ArchiveCleanup.Schedule, models.DeleteOldRepositoryArchives)
+               if err != nil {
+                       log.Fatal(4, "Cron[Clean up old repository archives]: %v", err)
+               }
+               if setting.Cron.ArchiveCleanup.RunAtStart {
+                       entry.Prev = time.Now()
+                       entry.ExecTimes++
+                       go models.DeleteOldRepositoryArchives()
+               }
+       }
        c.Start()
 }
 

diff --git a/modules/setting/setting.go b/modules/setting/setting.go
index 1cbb3eac0a2525822f78bb96a1689f964bf93674..747ddbf70839a93d755ce2aa26d374d2159b7e94 100644 (file)
--- a/modules/setting/setting.go
+++ b/modules/setting/setting.go
@@ -307,6 +307,12 @@ var (
                        RunAtStart bool
                        Schedule   string
                } `ini:"cron.check_repo_stats"`
+               ArchiveCleanup struct {
+                       Enabled    bool
+                       RunAtStart bool
+                       Schedule   string
+                       OlderThan  time.Duration
+               } `ini:"cron.archive_cleanup"`
        }{
                UpdateMirror: struct {
                        Enabled    bool
@@ -334,6 +340,16 @@ var (
                        RunAtStart: true,
                        Schedule:   "@every 24h",
                },
+               ArchiveCleanup: struct {
+                       Enabled    bool
+                       RunAtStart bool
+                       Schedule   string
+                       OlderThan  time.Duration
+               }{
+                       RunAtStart: true,
+                       Schedule:   "@every 24h",
+                       OlderThan:  24 * time.Hour,
+               },
        }
 
        // Git settings