From 985a39590ba07798dd6e6097e0c10401764c27fb Mon Sep 17 00:00:00 2001
From: Lunny Xiao <xiaolunwen@gmail.com>
Date: Sat, 21 Oct 2017 14:26:23 +0800
Subject: [PATCH] Use buffersize to reduce database connection when iterate
 (#2724)

* use buffersize to reduce database connection when iterate

* fix typo

* add default value on app.ini comment
---
 conf/app.ini               |  2 ++
 models/migrations/v19.go   |  2 +-
 models/migrations/v22.go   |  2 +-
 models/migrations/v26.go   |  2 +-
 models/repo.go             |  4 ++--
 models/user.go             |  2 +-
 modules/setting/setting.go | 20 +++++++++++---------
 7 files changed, 19 insertions(+), 15 deletions(-)

diff --git a/conf/app.ini b/conf/app.ini
index 471a4046f4..6524486b82 100644
--- a/conf/app.ini
+++ b/conf/app.ini
@@ -184,6 +184,8 @@ SSL_MODE = disable
 PATH = data/gitea.db
 ; For "sqlite3" only. Query timeout
 SQLITE_TIMEOUT = 500
+; For iterate buffer, default is 50
+ITERATE_BUFFER_SIZE = 50
 
 [indexer]
 ISSUE_INDEXER_PATH = indexers/issues.bleve
diff --git a/models/migrations/v19.go b/models/migrations/v19.go
index 0edeb3e92e..44338eb8ba 100644
--- a/models/migrations/v19.go
+++ b/models/migrations/v19.go
@@ -42,7 +42,7 @@ func generateAndMigrateGitHooks(x *xorm.Engine) (err error) {
 		}
 	)
 
-	return x.Where("id > 0").Iterate(new(Repository),
+	return x.Where("id > 0").BufferSize(setting.IterateBufferSize).Iterate(new(Repository),
 		func(idx int, bean interface{}) error {
 			repo := bean.(*Repository)
 			user := new(User)
diff --git a/models/migrations/v22.go b/models/migrations/v22.go
index 17709496e1..7774f6fd9a 100644
--- a/models/migrations/v22.go
+++ b/models/migrations/v22.go
@@ -42,7 +42,7 @@ func generateAndMigrateWikiGitHooks(x *xorm.Engine) (err error) {
 		}
 	)
 
-	return x.Where("id > 0").Iterate(new(Repository),
+	return x.Where("id > 0").BufferSize(setting.IterateBufferSize).Iterate(new(Repository),
 		func(idx int, bean interface{}) error {
 			repo := bean.(*Repository)
 			user := new(User)
diff --git a/models/migrations/v26.go b/models/migrations/v26.go
index 8b1c9a6326..636d1f76ca 100644
--- a/models/migrations/v26.go
+++ b/models/migrations/v26.go
@@ -36,7 +36,7 @@ func generateAndMigrateGitHookChains(x *xorm.Engine) (err error) {
 		hookTpl   = fmt.Sprintf("#!/usr/bin/env %s\ndata=$(cat)\nexitcodes=\"\"\nhookname=$(basename $0)\nGIT_DIR=${GIT_DIR:-$(dirname $0)}\n\nfor hook in ${GIT_DIR}/hooks/${hookname}.d/*; do\ntest -x \"${hook}\" || continue\necho \"${data}\" | \"${hook}\"\nexitcodes=\"${exitcodes} $?\"\ndone\n\nfor i in ${exitcodes}; do\n[ ${i} -eq 0 ] || exit ${i}\ndone\n", setting.ScriptType)
 	)
 
-	return x.Where("id > 0").Iterate(new(Repository),
+	return x.Where("id > 0").BufferSize(setting.IterateBufferSize).Iterate(new(Repository),
 		func(idx int, bean interface{}) error {
 			repo := bean.(*Repository)
 			user := new(User)
diff --git a/models/repo.go b/models/repo.go
index 293f251d3e..1b1be62f87 100644
--- a/models/repo.go
+++ b/models/repo.go
@@ -2145,7 +2145,7 @@ func GitFsck() {
 	log.Trace("Doing: GitFsck")
 
 	if err := x.
-		Where("id>0").
+		Where("id>0").BufferSize(setting.IterateBufferSize).
 		Iterate(new(Repository),
 			func(idx int, bean interface{}) error {
 				repo := bean.(*Repository)
@@ -2167,7 +2167,7 @@ func GitFsck() {
 func GitGcRepos() error {
 	args := append([]string{"gc"}, setting.Git.GCArgs...)
 	return x.
-		Where("id > 0").
+		Where("id > 0").BufferSize(setting.IterateBufferSize).
 		Iterate(new(Repository),
 			func(idx int, bean interface{}) error {
 				repo := bean.(*Repository)
diff --git a/models/user.go b/models/user.go
index f0cdc74d05..835a775ebc 100644
--- a/models/user.go
+++ b/models/user.go
@@ -816,7 +816,7 @@ func ChangeUserName(u *User, newUserName string) (err error) {
 	}
 
 	// Delete all local copies of repository wiki that user owns.
-	if err = x.
+	if err = x.BufferSize(setting.IterateBufferSize).
 		Where("owner_id=?", u.ID).
 		Iterate(new(Repository), func(idx int, bean interface{}) error {
 			repo := bean.(*Repository)
diff --git a/modules/setting/setting.go b/modules/setting/setting.go
index 10c301aa2a..0bd73b8cba 100644
--- a/modules/setting/setting.go
+++ b/modules/setting/setting.go
@@ -480,15 +480,16 @@ var (
 	ShowFooterTemplateLoadTime bool
 
 	// Global setting objects
-	Cfg           *ini.File
-	CustomPath    string // Custom directory path
-	CustomConf    string
-	CustomPID     string
-	ProdMode      bool
-	RunUser       string
-	IsWindows     bool
-	HasRobotsTxt  bool
-	InternalToken string // internal access token
+	Cfg               *ini.File
+	CustomPath        string // Custom directory path
+	CustomConf        string
+	CustomPID         string
+	ProdMode          bool
+	RunUser           string
+	IsWindows         bool
+	HasRobotsTxt      bool
+	InternalToken     string // internal access token
+	IterateBufferSize int
 )
 
 // DateLang transforms standard language locale name to corresponding value in datetime plugin.
@@ -873,6 +874,7 @@ func NewContext() {
 			log.Fatal(4, "Error saving generated JWT Secret to custom config: %v", err)
 		}
 	}
+	IterateBufferSize = Cfg.Section("database").Key("ITERATE_BUFFER_SIZE").MustInt(50)
 
 	sec = Cfg.Section("attachment")
 	AttachmentPath = sec.Key("PATH").MustString(path.Join(AppDataPath, "attachments"))
-- 
2.39.5