@@ -1719,6 +1719,12 @@ PATH = | |||
;RUN_AT_START = false | |||
;; Notice if not success | |||
;NO_SUCCESS_NOTICE = true | |||
;; Limit the number of mirrors added to the queue to this number | |||
;; (negative values mean no limit, 0 will result in no result in no mirrors being queued effectively disabling pull mirror updating.) | |||
;PULL_LIMIT=50 | |||
;; Limit the number of mirrors added to the queue to this number | |||
;; (negative values mean no limit, 0 will result in no mirrors being queued effectively disabling push mirror updating) | |||
;PUSH_LIMIT=50 | |||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; | |||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; |
@@ -828,6 +828,8 @@ NB: You must have `DISABLE_ROUTER_LOG` set to `false` for this option to take ef | |||
- `SCHEDULE`: **@every 10m**: Cron syntax for scheduling update mirrors, e.g. `@every 3h`. | |||
- `NO_SUCCESS_NOTICE`: **true**: The cron task for update mirrors success report is not very useful - as it just means that the mirrors have been queued. Therefore this is turned off by default. | |||
- `PULL_LIMIT`: **50**: Limit the number of mirrors added to the queue to this number (negative values mean no limit, 0 will result in no mirrors being queued effectively disabling pull mirror updating). | |||
- `PUSH_LIMIT`: **50**: Limit the number of mirrors added to the queue to this number (negative values mean no limit, 0 will result in no mirrors being queued effectively disabling push mirror updating). | |||
#### Cron - Repository Health Check (`cron.repo_health_check`) | |||
@@ -119,6 +119,7 @@ func MirrorsIterate(f func(idx int, bean interface{}) error) error { | |||
return db.GetEngine(db.DefaultContext). | |||
Where("next_update_unix<=?", time.Now().Unix()). | |||
And("next_update_unix!=0"). | |||
OrderBy("updated_unix ASC"). | |||
Iterate(new(Mirror), f) | |||
} | |||
@@ -107,5 +107,6 @@ func PushMirrorsIterate(f func(idx int, bean interface{}) error) error { | |||
return db.GetEngine(db.DefaultContext). | |||
Where("last_update + (`interval` / ?) <= ?", time.Second, time.Now().Unix()). | |||
And("`interval` != 0"). | |||
OrderBy("last_update ASC"). | |||
Iterate(new(PushMirror), f) | |||
} |
@@ -18,13 +18,24 @@ import ( | |||
) | |||
func registerUpdateMirrorTask() { | |||
RegisterTaskFatal("update_mirrors", &BaseConfig{ | |||
Enabled: true, | |||
RunAtStart: false, | |||
Schedule: "@every 10m", | |||
NoSuccessNotice: true, | |||
}, func(ctx context.Context, _ *models.User, _ Config) error { | |||
return mirror_service.Update(ctx) | |||
type UpdateMirrorTaskConfig struct { | |||
BaseConfig | |||
PullLimit int | |||
PushLimit int | |||
} | |||
RegisterTaskFatal("update_mirrors", &UpdateMirrorTaskConfig{ | |||
BaseConfig: BaseConfig{ | |||
Enabled: true, | |||
RunAtStart: false, | |||
Schedule: "@every 10m", | |||
NoSuccessNotice: true, | |||
}, | |||
PullLimit: 50, | |||
PushLimit: 50, | |||
}, func(ctx context.Context, _ *models.User, cfg Config) error { | |||
umtc := cfg.(*UpdateMirrorTaskConfig) | |||
return mirror_service.Update(ctx, umtc.PullLimit, umtc.PushLimit) | |||
}) | |||
} | |||
@@ -45,15 +45,19 @@ func doMirrorSync(ctx context.Context, req *SyncRequest) { | |||
} | |||
} | |||
var errLimit = fmt.Errorf("reached limit") | |||
// Update checks and updates mirror repositories. | |||
func Update(ctx context.Context) error { | |||
func Update(ctx context.Context, pullLimit, pushLimit int) error { | |||
if !setting.Mirror.Enabled { | |||
log.Warn("Mirror feature disabled, but cron job enabled: skip update") | |||
return nil | |||
} | |||
log.Trace("Doing: Update") | |||
handler := func(idx int, bean interface{}) error { | |||
requested := 0 | |||
handler := func(idx int, bean interface{}, limit int) error { | |||
var item SyncRequest | |||
if m, ok := bean.(*models.Mirror); ok { | |||
if m.Repo == nil { | |||
@@ -78,21 +82,49 @@ func Update(ctx context.Context) error { | |||
return nil | |||
} | |||
// Check we've not been cancelled | |||
select { | |||
case <-ctx.Done(): | |||
return fmt.Errorf("Aborted") | |||
return fmt.Errorf("aborted") | |||
default: | |||
return mirrorQueue.Push(&item) | |||
} | |||
// Check if this request is already in the queue | |||
has, err := mirrorQueue.Has(&item) | |||
if err != nil { | |||
return err | |||
} | |||
if has { | |||
return nil | |||
} | |||
// Push to the Queue | |||
if err := mirrorQueue.Push(&item); err != nil { | |||
return err | |||
} | |||
requested++ | |||
if limit > 0 && requested > limit { | |||
return errLimit | |||
} | |||
return nil | |||
} | |||
if err := models.MirrorsIterate(handler); err != nil { | |||
log.Error("MirrorsIterate: %v", err) | |||
return err | |||
if pullLimit != 0 { | |||
if err := models.MirrorsIterate(func(idx int, bean interface{}) error { | |||
return handler(idx, bean, pullLimit) | |||
}); err != nil && err != errLimit { | |||
log.Error("MirrorsIterate: %v", err) | |||
return err | |||
} | |||
} | |||
if err := models.PushMirrorsIterate(handler); err != nil { | |||
log.Error("PushMirrorsIterate: %v", err) | |||
return err | |||
if pushLimit != 0 { | |||
if err := models.PushMirrorsIterate(func(idx int, bean interface{}) error { | |||
return handler(idx, bean, pushLimit) | |||
}); err != nil && err != errLimit { | |||
log.Error("PushMirrorsIterate: %v", err) | |||
return err | |||
} | |||
} | |||
log.Trace("Finished: Update") | |||
return nil |