Calculate MAX_WORKERS default value by CPU number (go-gitea#26177)

To avoid consuming user's 100% CPU, limit the default value of
MAX_WORKERS

Fix go-gitea#26063 (the CPU 100% problem mentioned in it)

Author: wxiaoguang

custom/conf/app.example.ini

@@ -1414,8 +1414,8 @@ LEVEL = Info
 ;; Provides the suffix of the default redis/disk unique queue set name - specific queues can be overridden within in their [queue.name] sections.
 ;SET_NAME = "_unique"
 ;;
-;; Dynamically scale the worker pool to at this many workers
-;MAX_WORKERS = 10
+;; Maximum number of worker go-routines for the queue. Default value is "CpuNum/2" clipped to between 1 and 10.
+;MAX_WORKERS = ; (dynamic)
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

docs/content/administration/config-cheat-sheet.en-us.md

@@ -485,7 +485,7 @@ Configuration at `[queue]` will set defaults for queues with overrides for indiv
 - `CONN_STR`: **redis://127.0.0.1:6379/0**: Connection string for the redis queue type. For `redis-cluster` use `redis+cluster://127.0.0.1:6379/0`. Options can be set using query params. Similarly, LevelDB options can also be set using: **leveldb://relative/path?option=value** or **leveldb:///absolute/path?option=value**, and will override `DATADIR`
 - `QUEUE_NAME`: **_queue**: The suffix for default redis and disk queue name. Individual queues will default to **`name`**`QUEUE_NAME` but can be overridden in the specific `queue.name` section.
 - `SET_NAME`: **_unique**: The suffix that will be added to the default redis and disk queue `set` name for unique queues. Individual queues will default to **`name`**`QUEUE_NAME`_`SET_NAME`_ but can be overridden in the specific `queue.name` section.
-- `MAX_WORKERS`: **10**: Maximum number of worker go-routines for the queue.
+- `MAX_WORKERS`: **(dynamic)**: Maximum number of worker go-routines for the queue. Default value is "CpuNum/2" clipped to between 1 and 10.
 
 Gitea creates the following non-unique queues:
 

modules/queue/manager_test.go

@@ -51,7 +51,7 @@ CONN_STR = redis://
 	assert.Equal(t, "", q.baseConfig.ConnStr)
 	assert.Equal(t, "default_queue", q.baseConfig.QueueFullName)
 	assert.Equal(t, "default_queue_unique", q.baseConfig.SetFullName)
-	assert.Equal(t, 10, q.GetWorkerMaxNumber())
+	assert.NotZero(t, q.GetWorkerMaxNumber())
 	assert.Equal(t, 0, q.GetWorkerNumber())
 	assert.Equal(t, 0, q.GetWorkerActiveNumber())
 	assert.Equal(t, 0, q.GetQueueItemNumber())
@@ -75,7 +75,7 @@ BATCH_LENGTH = 22
 CONN_STR =
 QUEUE_NAME = _q2
 SET_NAME = _u2
-MAX_WORKERS = 2
+MAX_WORKERS = 123
 `)
 
 	assert.NoError(t, err)
@@ -89,7 +89,7 @@ MAX_WORKERS = 2
 	assert.Equal(t, "addrs=127.0.0.1:6379 db=0", q1.baseConfig.ConnStr)
 	assert.Equal(t, "no-such_queue1", q1.baseConfig.QueueFullName)
 	assert.Equal(t, "no-such_queue1_unique", q1.baseConfig.SetFullName)
-	assert.Equal(t, 10, q1.GetWorkerMaxNumber())
+	assert.NotZero(t, q1.GetWorkerMaxNumber())
 	assert.Equal(t, 0, q1.GetWorkerNumber())
 	assert.Equal(t, 0, q1.GetWorkerActiveNumber())
 	assert.Equal(t, 0, q1.GetQueueItemNumber())
@@ -105,7 +105,7 @@ MAX_WORKERS = 2
 	assert.Equal(t, "", q2.baseConfig.ConnStr)
 	assert.Equal(t, "sub_q2", q2.baseConfig.QueueFullName)
 	assert.Equal(t, "sub_q2_u2", q2.baseConfig.SetFullName)
-	assert.Equal(t, 2, q2.GetWorkerMaxNumber())
+	assert.Equal(t, 123, q2.GetWorkerMaxNumber())
 	assert.Equal(t, 0, q2.GetWorkerNumber())
 	assert.Equal(t, 0, q2.GetWorkerActiveNumber())
 	assert.Equal(t, 0, q2.GetQueueItemNumber())

modules/setting/queue.go

@@ -5,6 +5,7 @@ package setting
 
 import (
 	"path/filepath"
+	"runtime"
 
 	"code.gitea.io/gitea/modules/json"
 	"code.gitea.io/gitea/modules/log"
@@ -25,18 +26,24 @@ type QueueSettings struct {
 	MaxWorkers  int
 }
 
-var queueSettingsDefault = QueueSettings{
-	Type:    "level",         // dummy, channel, level, redis
-	Datadir: "queues/common", // relative to AppDataPath
-	Length:  100,             // queue length before a channel queue will block
-
-	QueueName:   "_queue",
-	SetName:     "_unique",
-	BatchLength: 20,
-	MaxWorkers:  10,
-}
-
 func GetQueueSettings(rootCfg ConfigProvider, name string) (QueueSettings, error) {
+	queueSettingsDefault := QueueSettings{
+		Type:    "level",         // dummy, channel, level, redis
+		Datadir: "queues/common", // relative to AppDataPath
+		Length:  100,             // queue length before a channel queue will block
+
+		QueueName:   "_queue",
+		SetName:     "_unique",
+		BatchLength: 20,
+		MaxWorkers:  runtime.NumCPU() / 2,
+	}
+	if queueSettingsDefault.MaxWorkers < 1 {
+		queueSettingsDefault.MaxWorkers = 1
+	}
+	if queueSettingsDefault.MaxWorkers > 10 {
+		queueSettingsDefault.MaxWorkers = 10
+	}
+
 	// deep copy default settings
 	cfg := QueueSettings{}
 	if cfgBs, err := json.Marshal(queueSettingsDefault); err != nil {