Use buffersize to reduce database connection when iterate (#2724)

* use buffersize to reduce database connection when iterate

* fix typo

* add default value on app.ini comment

diff --git a/conf/app.ini b/conf/app.ini
index 471a4046f461c36870ce5312dc47565ea35f7adc..6524486b827ea00aa5e2e1a5dbdb4a6e9829ba5c 100644 (file)
--- a/conf/app.ini
+++ b/conf/app.ini
@@ -184,6 +184,8 @@ SSL_MODE = disable
 PATH = data/gitea.db
 ; For "sqlite3" only. Query timeout
 SQLITE_TIMEOUT = 500
+; For iterate buffer, default is 50
+ITERATE_BUFFER_SIZE = 50
 
 [indexer]
 ISSUE_INDEXER_PATH = indexers/issues.bleve

diff --git a/models/migrations/v19.go b/models/migrations/v19.go
index 0edeb3e92e46fa776947d494309af596d0f8d77e..44338eb8ba2580cc13bc1d2fd5357ba34e293455 100644 (file)
--- a/models/migrations/v19.go
+++ b/models/migrations/v19.go
@@ -42,7 +42,7 @@ func generateAndMigrateGitHooks(x *xorm.Engine) (err error) {
                }
        )
 
-       return x.Where("id > 0").Iterate(new(Repository),
+       return x.Where("id > 0").BufferSize(setting.IterateBufferSize).Iterate(new(Repository),
                func(idx int, bean interface{}) error {
                        repo := bean.(*Repository)
                        user := new(User)

diff --git a/models/migrations/v22.go b/models/migrations/v22.go
index 17709496e16bbd0004e3d3533fbad5d3d0bd9de4..7774f6fd9a977ce1e35f0bc6722cfc4635865369 100644 (file)
--- a/models/migrations/v22.go
+++ b/models/migrations/v22.go
@@ -42,7 +42,7 @@ func generateAndMigrateWikiGitHooks(x *xorm.Engine) (err error) {
                }
        )
 
-       return x.Where("id > 0").Iterate(new(Repository),
+       return x.Where("id > 0").BufferSize(setting.IterateBufferSize).Iterate(new(Repository),
                func(idx int, bean interface{}) error {
                        repo := bean.(*Repository)
                        user := new(User)

diff --git a/models/migrations/v26.go b/models/migrations/v26.go
index 8b1c9a6326f44ea8482e6057eff9268db35407eb..636d1f76ca9600e209ae05206e053fd341cb4ab8 100644 (file)
--- a/models/migrations/v26.go
+++ b/models/migrations/v26.go
@@ -36,7 +36,7 @@ func generateAndMigrateGitHookChains(x *xorm.Engine) (err error) {
                hookTpl   = fmt.Sprintf("#!/usr/bin/env %s\ndata=$(cat)\nexitcodes=\"\"\nhookname=$(basename $0)\nGIT_DIR=${GIT_DIR:-$(dirname $0)}\n\nfor hook in ${GIT_DIR}/hooks/${hookname}.d/*; do\ntest -x \"${hook}\" || continue\necho \"${data}\" | \"${hook}\"\nexitcodes=\"${exitcodes} $?\"\ndone\n\nfor i in ${exitcodes}; do\n[ ${i} -eq 0 ] || exit ${i}\ndone\n", setting.ScriptType)
        )
 
-       return x.Where("id > 0").Iterate(new(Repository),
+       return x.Where("id > 0").BufferSize(setting.IterateBufferSize).Iterate(new(Repository),
                func(idx int, bean interface{}) error {
                        repo := bean.(*Repository)
                        user := new(User)

diff --git a/models/repo.go b/models/repo.go
index 293f251d3e08b1c34febfbfecc0c8c8806c9de6e..1b1be62f875d4236c734c81161a98af902eb2af6 100644 (file)
--- a/models/repo.go
+++ b/models/repo.go
@@ -2145,7 +2145,7 @@ func GitFsck() {
        log.Trace("Doing: GitFsck")
 
        if err := x.
-               Where("id>0").
+               Where("id>0").BufferSize(setting.IterateBufferSize).
                Iterate(new(Repository),
                        func(idx int, bean interface{}) error {
                                repo := bean.(*Repository)
@@ -2167,7 +2167,7 @@ func GitFsck() {
 func GitGcRepos() error {
        args := append([]string{"gc"}, setting.Git.GCArgs...)
        return x.
-               Where("id > 0").
+               Where("id > 0").BufferSize(setting.IterateBufferSize).
                Iterate(new(Repository),
                        func(idx int, bean interface{}) error {
                                repo := bean.(*Repository)

diff --git a/models/user.go b/models/user.go
index f0cdc74d05f28356590212f74b37a08b0d375779..835a775ebcba930cc1f695887181b44621946424 100644 (file)
--- a/models/user.go
+++ b/models/user.go
@@ -816,7 +816,7 @@ func ChangeUserName(u *User, newUserName string) (err error) {
        }
 
        // Delete all local copies of repository wiki that user owns.
-       if err = x.
+       if err = x.BufferSize(setting.IterateBufferSize).
                Where("owner_id=?", u.ID).
                Iterate(new(Repository), func(idx int, bean interface{}) error {
                        repo := bean.(*Repository)

diff --git a/modules/setting/setting.go b/modules/setting/setting.go
index 10c301aa2af3930719e5401e6ebbaaf077bb59a6..0bd73b8cba51ef7dca31d60f8a69ccf3a45842cd 100644 (file)
--- a/modules/setting/setting.go
+++ b/modules/setting/setting.go
@@ -480,15 +480,16 @@ var (
        ShowFooterTemplateLoadTime bool
 
        // Global setting objects
-       Cfg           *ini.File
-       CustomPath    string // Custom directory path
-       CustomConf    string
-       CustomPID     string
-       ProdMode      bool
-       RunUser       string
-       IsWindows     bool
-       HasRobotsTxt  bool
-       InternalToken string // internal access token
+       Cfg               *ini.File
+       CustomPath        string // Custom directory path
+       CustomConf        string
+       CustomPID         string
+       ProdMode          bool
+       RunUser           string
+       IsWindows         bool
+       HasRobotsTxt      bool
+       InternalToken     string // internal access token
+       IterateBufferSize int
 )
 
 // DateLang transforms standard language locale name to corresponding value in datetime plugin.
@@ -873,6 +874,7 @@ func NewContext() {
                        log.Fatal(4, "Error saving generated JWT Secret to custom config: %v", err)
                }
        }
+       IterateBufferSize = Cfg.Section("database").Key("ITERATE_BUFFER_SIZE").MustInt(50)
 
        sec = Cfg.Section("attachment")
        AttachmentPath = sec.Key("PATH").MustString(path.Join(AppDataPath, "attachments"))