To avoid consuming user's 100% CPU, limit the default value of MAX_WORKERS Fix #26063 (the CPU 100% problem mentioned in it)tags/v1.21.0-rc0
@@ -1420,8 +1420,8 @@ LEVEL = Info | |||
;; Provides the suffix of the default redis/disk unique queue set name - specific queues can be overridden within in their [queue.name] sections. | |||
;SET_NAME = "_unique" | |||
;; | |||
;; Dynamically scale the worker pool to at this many workers | |||
;MAX_WORKERS = 10 | |||
;; Maximum number of worker go-routines for the queue. Default value is "CpuNum/2" clipped to between 1 and 10. | |||
;MAX_WORKERS = ; (dynamic) | |||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; | |||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; |
@@ -488,7 +488,7 @@ Configuration at `[queue]` will set defaults for queues with overrides for indiv | |||
- `CONN_STR`: **redis://127.0.0.1:6379/0**: Connection string for the redis queue type. For `redis-cluster` use `redis+cluster://127.0.0.1:6379/0`. Options can be set using query params. Similarly, LevelDB options can also be set using: **leveldb://relative/path?option=value** or **leveldb:///absolute/path?option=value**, and will override `DATADIR` | |||
- `QUEUE_NAME`: **_queue**: The suffix for default redis and disk queue name. Individual queues will default to **`name`**`QUEUE_NAME` but can be overridden in the specific `queue.name` section. | |||
- `SET_NAME`: **_unique**: The suffix that will be added to the default redis and disk queue `set` name for unique queues. Individual queues will default to **`name`**`QUEUE_NAME`_`SET_NAME`_ but can be overridden in the specific `queue.name` section. | |||
- `MAX_WORKERS`: **10**: Maximum number of worker go-routines for the queue. | |||
- `MAX_WORKERS`: **(dynamic)**: Maximum number of worker go-routines for the queue. Default value is "CpuNum/2" clipped to between 1 and 10. | |||
Gitea creates the following non-unique queues: | |||
@@ -51,7 +51,7 @@ CONN_STR = redis:// | |||
assert.Equal(t, "", q.baseConfig.ConnStr) | |||
assert.Equal(t, "default_queue", q.baseConfig.QueueFullName) | |||
assert.Equal(t, "default_queue_unique", q.baseConfig.SetFullName) | |||
assert.Equal(t, 10, q.GetWorkerMaxNumber()) | |||
assert.NotZero(t, q.GetWorkerMaxNumber()) | |||
assert.Equal(t, 0, q.GetWorkerNumber()) | |||
assert.Equal(t, 0, q.GetWorkerActiveNumber()) | |||
assert.Equal(t, 0, q.GetQueueItemNumber()) | |||
@@ -75,7 +75,7 @@ BATCH_LENGTH = 22 | |||
CONN_STR = | |||
QUEUE_NAME = _q2 | |||
SET_NAME = _u2 | |||
MAX_WORKERS = 2 | |||
MAX_WORKERS = 123 | |||
`) | |||
assert.NoError(t, err) | |||
@@ -89,7 +89,7 @@ MAX_WORKERS = 2 | |||
assert.Equal(t, "addrs=127.0.0.1:6379 db=0", q1.baseConfig.ConnStr) | |||
assert.Equal(t, "no-such_queue1", q1.baseConfig.QueueFullName) | |||
assert.Equal(t, "no-such_queue1_unique", q1.baseConfig.SetFullName) | |||
assert.Equal(t, 10, q1.GetWorkerMaxNumber()) | |||
assert.NotZero(t, q1.GetWorkerMaxNumber()) | |||
assert.Equal(t, 0, q1.GetWorkerNumber()) | |||
assert.Equal(t, 0, q1.GetWorkerActiveNumber()) | |||
assert.Equal(t, 0, q1.GetQueueItemNumber()) | |||
@@ -105,7 +105,7 @@ MAX_WORKERS = 2 | |||
assert.Equal(t, "", q2.baseConfig.ConnStr) | |||
assert.Equal(t, "sub_q2", q2.baseConfig.QueueFullName) | |||
assert.Equal(t, "sub_q2_u2", q2.baseConfig.SetFullName) | |||
assert.Equal(t, 2, q2.GetWorkerMaxNumber()) | |||
assert.Equal(t, 123, q2.GetWorkerMaxNumber()) | |||
assert.Equal(t, 0, q2.GetWorkerNumber()) | |||
assert.Equal(t, 0, q2.GetWorkerActiveNumber()) | |||
assert.Equal(t, 0, q2.GetQueueItemNumber()) |
@@ -5,6 +5,7 @@ package setting | |||
import ( | |||
"path/filepath" | |||
"runtime" | |||
"code.gitea.io/gitea/modules/json" | |||
"code.gitea.io/gitea/modules/log" | |||
@@ -25,18 +26,24 @@ type QueueSettings struct { | |||
MaxWorkers int | |||
} | |||
var queueSettingsDefault = QueueSettings{ | |||
Type: "level", // dummy, channel, level, redis | |||
Datadir: "queues/common", // relative to AppDataPath | |||
Length: 100, // queue length before a channel queue will block | |||
QueueName: "_queue", | |||
SetName: "_unique", | |||
BatchLength: 20, | |||
MaxWorkers: 10, | |||
} | |||
func GetQueueSettings(rootCfg ConfigProvider, name string) (QueueSettings, error) { | |||
queueSettingsDefault := QueueSettings{ | |||
Type: "level", // dummy, channel, level, redis | |||
Datadir: "queues/common", // relative to AppDataPath | |||
Length: 100, // queue length before a channel queue will block | |||
QueueName: "_queue", | |||
SetName: "_unique", | |||
BatchLength: 20, | |||
MaxWorkers: runtime.NumCPU() / 2, | |||
} | |||
if queueSettingsDefault.MaxWorkers < 1 { | |||
queueSettingsDefault.MaxWorkers = 1 | |||
} | |||
if queueSettingsDefault.MaxWorkers > 10 { | |||
queueSettingsDefault.MaxWorkers = 10 | |||
} | |||
// deep copy default settings | |||
cfg := QueueSettings{} | |||
if cfgBs, err := json.Marshal(queueSettingsDefault); err != nil { |