Add repo-sync-releases admin command (#3254)

* Add repo-sync-releases admin command

Will help recovering corrupted database, see #3247

* Load repos in chunks of 10, exit with error if unable to get a list, scan private repos, fix typo

* Fix debug output about num releases

* Introduce RepositoryListDefaultPageSize constant, set to 64

Use it from the new admin command

* Use RepositoryListDefaultPageSize in more places

* Document RepositoryListDefaultPageSize

diff --git a/cmd/admin.go b/cmd/admin.go
index 928bc1ebba466984d7ce2753b7f7a1b0e982af4d..4df28358a2f7311b773a8011b768a47af5810f41 100644 (file)
--- a/cmd/admin.go
+++ b/cmd/admin.go
@@ -8,7 +8,9 @@ package cmd
 import (
        "fmt"
 
+       "code.gitea.io/git"
        "code.gitea.io/gitea/models"
+       "code.gitea.io/gitea/modules/log"
        "code.gitea.io/gitea/modules/setting"
 
        "github.com/urfave/cli"
@@ -24,6 +26,7 @@ to make automatic initialization process more smoothly`,
                Subcommands: []cli.Command{
                        subcmdCreateUser,
                        subcmdChangePassword,
+                       subcmdRepoSyncReleases,
                },
        }
 
@@ -76,6 +79,12 @@ to make automatic initialization process more smoothly`,
                        },
                },
        }
+
+       subcmdRepoSyncReleases = cli.Command{
+               Name:   "repo-sync-releases",
+               Usage:  "Synchronize repository releases with tags",
+               Action: runRepoSyncReleases,
+       }
 )
 
 func runChangePassword(c *cli.Context) error {
@@ -145,3 +154,69 @@ func runCreateUser(c *cli.Context) error {
        fmt.Printf("New user '%s' has been successfully created!\n", c.String("name"))
        return nil
 }
+
+func runRepoSyncReleases(c *cli.Context) error {
+
+       setting.NewContext()
+       models.LoadConfigs()
+
+       setting.NewXORMLogService(false)
+       if err := models.SetEngine(); err != nil {
+               return fmt.Errorf("models.SetEngine: %v", err)
+       }
+
+       log.Trace("Synchronizing repository releases (this may take a while)")
+       for page := 1; ; page++ {
+               repos, count, err := models.SearchRepositoryByName(&models.SearchRepoOptions{
+                       Page:     page,
+                       PageSize: models.RepositoryListDefaultPageSize,
+                       Private:  true,
+               })
+               if err != nil {
+                       log.Fatal(4, "SearchRepositoryByName: %v", err)
+                       return err
+               }
+               if len(repos) == 0 {
+                       break
+               }
+               log.Trace("Processing next %d repos of %d", len(repos), count)
+               for _, repo := range repos {
+                       log.Trace("Synchronizing repo %s with path %s", repo.FullName(), repo.RepoPath())
+                       gitRepo, err := git.OpenRepository(repo.RepoPath())
+                       if err != nil {
+                               log.Warn("OpenRepository: %v", err)
+                               continue
+                       }
+
+                       oldnum, err := models.GetReleaseCountByRepoID(repo.ID,
+                               models.FindReleasesOptions{
+                                       IncludeDrafts: false,
+                                       IncludeTags:   true,
+                               })
+                       if err != nil {
+                               log.Warn(" GetReleaseCountByRepoID: %v", err)
+                       }
+                       log.Trace(" currentNumReleases is %d, running SyncReleasesWithTags", oldnum)
+
+                       if err = models.SyncReleasesWithTags(repo, gitRepo); err != nil {
+                               log.Warn(" SyncReleasesWithTags: %v", err)
+                               continue
+                       }
+
+                       count, err = models.GetReleaseCountByRepoID(repo.ID,
+                               models.FindReleasesOptions{
+                                       IncludeDrafts: false,
+                                       IncludeTags:   true,
+                               })
+                       if err != nil {
+                               log.Warn(" GetReleaseCountByRepoID: %v", err)
+                               continue
+                       }
+
+                       log.Trace(" repo %s releases synchronized to tags: from %d to %d",
+                               repo.FullName(), oldnum, count)
+               }
+       }
+
+       return nil
+}

diff --git a/models/issue_indexer.go b/models/issue_indexer.go
index 26d053a5d710ba0eb875a30125a1d1a7cc6d4c36..922b66f95effd6997281604e3536584448e1b1fc 100644 (file)
--- a/models/issue_indexer.go
+++ b/models/issue_indexer.go
@@ -29,7 +29,7 @@ func populateIssueIndexer() error {
        for page := 1; ; page++ {
                repos, _, err := SearchRepositoryByName(&SearchRepoOptions{
                        Page:        page,
-                       PageSize:    10,
+                       PageSize:    RepositoryListDefaultPageSize,
                        OrderBy:     SearchOrderByID,
                        Private:     true,
                        Collaborate: util.OptionalBoolFalse,

diff --git a/models/migrations/v39.go b/models/migrations/v39.go
index 3547ef1f9e0d1e0f3ac7e7f89ad8804bc05399cc..42197e80f282f681cc3db7a0a68ff81459d7070b 100644 (file)
--- a/models/migrations/v39.go
+++ b/models/migrations/v39.go
@@ -31,7 +31,7 @@ func releaseAddColumnIsTagAndSyncTags(x *xorm.Engine) error {
 
        // For the sake of SQLite3, we can't use x.Iterate here.
        offset := 0
-       pageSize := 20
+       pageSize := models.RepositoryListDefaultPageSize
        for {
                repos := make([]*models.Repository, 0, pageSize)
                if err := x.Table("repository").Asc("id").Limit(pageSize, offset).Find(&repos); err != nil {

diff --git a/models/repo_indexer.go b/models/repo_indexer.go
index 4877d339f8cc249269c3bf69dadb63b62c138abb..8cc08904ed4bd8aaa814978b904705b2a64262b0 100644 (file)
--- a/models/repo_indexer.go
+++ b/models/repo_indexer.go
@@ -81,7 +81,7 @@ func populateRepoIndexer() error {
        for page := 1; ; page++ {
                repos, _, err := SearchRepositoryByName(&SearchRepoOptions{
                        Page:     page,
-                       PageSize: 10,
+                       PageSize: RepositoryListDefaultPageSize,
                        OrderBy:  SearchOrderByID,
                        Private:  true,
                })

diff --git a/models/repo_list.go b/models/repo_list.go
index d13266f2087d3e077fa78ef167027cca9acfea76..bc9b831d30413a399fb5eca31fdb2c0d6f5379dc 100644 (file)
--- a/models/repo_list.go
+++ b/models/repo_list.go
@@ -13,6 +13,13 @@ import (
        "github.com/go-xorm/builder"
 )
 
+// RepositoryListDefaultPageSize is the default number of repositories
+// to load in memory when running administrative tasks on all (or almost
+// all) of them.
+// The number should be low enough to avoid filling up all RAM with
+// repository data...
+const RepositoryListDefaultPageSize = 64
+
 // RepositoryList contains a list of repositories
 type RepositoryList []*Repository