Add retry for migration http/https requests (#9019)

* Add retry for migration http/https requests

* give the more suitable name for retry configuraion items

* fix docs and lint

* Only use retryDownloader when setting > 1

diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample
index 34c3ee9db5466a7c543addc241919af4a3a3a38d..599498a4b62c5afa359ba6f2654fb1f404e6b8ab 100644 (file)
--- a/custom/conf/app.ini.sample
+++ b/custom/conf/app.ini.sample
@@ -892,3 +892,9 @@ QUEUE_LENGTH = 1000
 ; Task queue connection string, available only when `QUEUE_TYPE` is `redis`.
 ; If there is a password of redis, use `addrs=127.0.0.1:6379 password=123 db=0`.
 QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"
+
+[migrations]
+; Max attempts per http/https request on migrations.
+MAX_ATTEMPTS = 3
+; Backoff time per http/https request retry (seconds)
+RETRY_BACKOFF = 3
\ No newline at end of file

diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md
index ab353f9d5aa6f2a1a7d017b2ae5c00cc3d89e2d3..f56ed34e431e4dfa96daf6496feb32ae4c4375ce 100644 (file)
--- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md
+++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md
@@ -585,6 +585,11 @@ Two special environment variables are passed to the render command:
 - `QUEUE_LENGTH`: **1000**: Task queue length, available only when `QUEUE_TYPE` is `channel`.
 - `QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: Task queue connection string, available only when `QUEUE_TYPE` is `redis`. If there redis needs a password, use `addrs=127.0.0.1:6379 password=123 db=0`.
 
+## Migrations (`migrations`)
+
+- `MAX_ATTEMPTS`: **3**: Max attempts per http/https request on migrations.
+- `RETRY_BACKOFF`: **3**: Backoff time per http/https request retry (seconds)
+
 ## Other (`other`)
 
 - `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer.

diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md
index b545d9a99d1f3129eefac6b7d448b755e5b215da..22e47f810f6dfd152d7d72c8f6d1d5263a163dd4 100644 (file)
--- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md
+++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md
@@ -272,6 +272,11 @@ IS_INPUT_FILE = false
 - `QUEUE_LENGTH`: **1000**: 任务队列长度，当 `QUEUE_TYPE` 为 `channel` 时有效。
 - `QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: 任务队列连接字符串，当 `QUEUE_TYPE` 为 `redis` 时有效。如果redis有密码，则可以 `addrs=127.0.0.1:6379 password=123 db=0`。
 
+## Migrations (`migrations`)
+
+- `MAX_ATTEMPTS`: **3**: 在迁移过程中的 http/https 请求重试次数。
+- `RETRY_BACKOFF`: **3**: 等待下一次重试的时间，单位秒。
+
 ## Other (`other`)
 
 - `SHOW_FOOTER_BRANDING`: 为真则在页面底部显示Gitea的字样。

diff --git a/modules/migrations/base/downloader.go b/modules/migrations/base/downloader.go
index 69c2adb9e95035ceda20ec4d0eb86489b5b61436..b853ec30209a1365fa74a99e256056d3aec4d9aa 100644 (file)
--- a/modules/migrations/base/downloader.go
+++ b/modules/migrations/base/downloader.go
@@ -5,7 +5,11 @@
 
 package base
 
-import "code.gitea.io/gitea/modules/structs"
+import (
+       "time"
+
+       "code.gitea.io/gitea/modules/structs"
+)
 
 // Downloader downloads the site repo informations
 type Downloader interface {
@@ -25,3 +29,148 @@ type DownloaderFactory interface {
        New(opts MigrateOptions) (Downloader, error)
        GitServiceType() structs.GitServiceType
 }
+
+// RetryDownloader retry the downloads
+type RetryDownloader struct {
+       Downloader
+       RetryTimes int // the total execute times
+       RetryDelay int // time to delay seconds
+}
+
+// NewRetryDownloader creates a retry downloader
+func NewRetryDownloader(downloader Downloader, retryTimes, retryDelay int) *RetryDownloader {
+       return &RetryDownloader{
+               Downloader: downloader,
+               RetryTimes: retryTimes,
+               RetryDelay: retryDelay,
+       }
+}
+
+// GetRepoInfo returns a repository information with retry
+func (d *RetryDownloader) GetRepoInfo() (*Repository, error) {
+       var (
+               times = d.RetryTimes
+               repo  *Repository
+               err   error
+       )
+       for ; times > 0; times-- {
+               if repo, err = d.Downloader.GetRepoInfo(); err == nil {
+                       return repo, nil
+               }
+               time.Sleep(time.Second * time.Duration(d.RetryDelay))
+       }
+       return nil, err
+}
+
+// GetTopics returns a repository's topics with retry
+func (d *RetryDownloader) GetTopics() ([]string, error) {
+       var (
+               times  = d.RetryTimes
+               topics []string
+               err    error
+       )
+       for ; times > 0; times-- {
+               if topics, err = d.Downloader.GetTopics(); err == nil {
+                       return topics, nil
+               }
+               time.Sleep(time.Second * time.Duration(d.RetryDelay))
+       }
+       return nil, err
+}
+
+// GetMilestones returns a repository's milestones with retry
+func (d *RetryDownloader) GetMilestones() ([]*Milestone, error) {
+       var (
+               times      = d.RetryTimes
+               milestones []*Milestone
+               err        error
+       )
+       for ; times > 0; times-- {
+               if milestones, err = d.Downloader.GetMilestones(); err == nil {
+                       return milestones, nil
+               }
+               time.Sleep(time.Second * time.Duration(d.RetryDelay))
+       }
+       return nil, err
+}
+
+// GetReleases returns a repository's releases with retry
+func (d *RetryDownloader) GetReleases() ([]*Release, error) {
+       var (
+               times    = d.RetryTimes
+               releases []*Release
+               err      error
+       )
+       for ; times > 0; times-- {
+               if releases, err = d.Downloader.GetReleases(); err == nil {
+                       return releases, nil
+               }
+               time.Sleep(time.Second * time.Duration(d.RetryDelay))
+       }
+       return nil, err
+}
+
+// GetLabels returns a repository's labels with retry
+func (d *RetryDownloader) GetLabels() ([]*Label, error) {
+       var (
+               times  = d.RetryTimes
+               labels []*Label
+               err    error
+       )
+       for ; times > 0; times-- {
+               if labels, err = d.Downloader.GetLabels(); err == nil {
+                       return labels, nil
+               }
+               time.Sleep(time.Second * time.Duration(d.RetryDelay))
+       }
+       return nil, err
+}
+
+// GetIssues returns a repository's issues with retry
+func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) {
+       var (
+               times  = d.RetryTimes
+               issues []*Issue
+               isEnd  bool
+               err    error
+       )
+       for ; times > 0; times-- {
+               if issues, isEnd, err = d.Downloader.GetIssues(page, perPage); err == nil {
+                       return issues, isEnd, nil
+               }
+               time.Sleep(time.Second * time.Duration(d.RetryDelay))
+       }
+       return nil, false, err
+}
+
+// GetComments returns a repository's comments with retry
+func (d *RetryDownloader) GetComments(issueNumber int64) ([]*Comment, error) {
+       var (
+               times    = d.RetryTimes
+               comments []*Comment
+               err      error
+       )
+       for ; times > 0; times-- {
+               if comments, err = d.Downloader.GetComments(issueNumber); err == nil {
+                       return comments, nil
+               }
+               time.Sleep(time.Second * time.Duration(d.RetryDelay))
+       }
+       return nil, err
+}
+
+// GetPullRequests returns a repository's pull requests with retry
+func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, error) {
+       var (
+               times = d.RetryTimes
+               prs   []*PullRequest
+               err   error
+       )
+       for ; times > 0; times-- {
+               if prs, err = d.Downloader.GetPullRequests(page, perPage); err == nil {
+                       return prs, nil
+               }
+               time.Sleep(time.Second * time.Duration(d.RetryDelay))
+       }
+       return nil, err
+}

diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go
index 7a5071e1258dd7b0684786aa2ebddecbf09b9f85..cfaedf78e3504812985a2cf1d0d7d6e685418c99 100644 (file)
--- a/modules/migrations/migrate.go
+++ b/modules/migrations/migrate.go
@@ -11,6 +11,7 @@ import (
        "code.gitea.io/gitea/models"
        "code.gitea.io/gitea/modules/log"
        "code.gitea.io/gitea/modules/migrations/base"
+       "code.gitea.io/gitea/modules/setting"
        "code.gitea.io/gitea/modules/structs"
 )
 
@@ -63,6 +64,11 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt
        }
 
        uploader.gitServiceType = opts.GitServiceType
+
+       if setting.Migrations.MaxAttempts > 1 {
+               downloader = base.NewRetryDownloader(downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff)
+       }
+
        if err := migrateRepository(downloader, uploader, opts); err != nil {
                if err1 := uploader.Rollback(); err1 != nil {
                        log.Error("rollback failed: %v", err1)

diff --git a/modules/setting/migrations.go b/modules/setting/migrations.go
new file mode 100644 (file)
index 0000000..51d6bbc
--- /dev/null
+++ b/modules/setting/migrations.go
@@ -0,0 +1,22 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package setting
+
+var (
+       // Migrations settings
+       Migrations = struct {
+               MaxAttempts  int
+               RetryBackoff int
+       }{
+               MaxAttempts:  3,
+               RetryBackoff: 3,
+       }
+)
+
+func newMigrationsService() {
+       sec := Cfg.Section("migrations")
+       Migrations.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(Migrations.MaxAttempts)
+       Migrations.RetryBackoff = sec.Key("RETRY_BACKOFF").MustInt(Migrations.RetryBackoff)
+}

diff --git a/modules/setting/setting.go b/modules/setting/setting.go
index c0b9b99e3cbbeeea80e15910ed10a666ede6e496..6a45797cf2ebde07a92021e3f995c20dcea80676 100644 (file)
--- a/modules/setting/setting.go
+++ b/modules/setting/setting.go
@@ -1062,6 +1062,7 @@ func NewServices() {
        newRegisterMailService()
        newNotifyMailService()
        newWebhookService()
+       newMigrationsService()
        newIndexerService()
        newTaskService()
 }