From 62eb1b0f2530a5ae1ce9b729378c0c8066174215 Mon Sep 17 00:00:00 2001 From: zeripath Date: Tue, 7 Jan 2020 11:23:09 +0000 Subject: Graceful Queues: Issue Indexing and Tasks (#9363) * Queue: Add generic graceful queues with settings * Queue & Setting: Add worker pool implementation * Queue: Add worker settings * Queue: Make resizing worker pools * Queue: Add name variable to queues * Queue: Add monitoring * Queue: Improve logging * Issues: Gracefulise the issues indexer Remove the old now unused specific queues * Task: Move to generic queue and gracefulise * Issues: Standardise the issues indexer queue settings * Fix test * Queue: Allow Redis to connect to unix * Prevent deadlock during early shutdown of issue indexer * Add MaxWorker settings to queues * Merge branch 'master' into graceful-queues * Update modules/indexer/issues/indexer.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update modules/indexer/issues/indexer.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update modules/queue/queue_channel.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update modules/queue/queue_disk.go * Update modules/queue/queue_disk_channel.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Rename queue.Description to queue.ManagedQueue as per @guillep2k * Cancel pool workers when removed * Remove dependency on queue from setting * Update modules/queue/queue_redis.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * As per @guillep2k add mutex locks on shutdown/terminate * move unlocking out of setInternal * Add warning if number of workers < 0 * Small changes as per @guillep2k * No redis host specified not found * Clean up documentation for queues * Update docs/content/doc/advanced/config-cheat-sheet.en-us.md * Update modules/indexer/issues/indexer_test.go * Ensure that persistable channel queue is added to manager * Rename QUEUE_NAME REDIS_QUEUE_NAME * Revert "Rename QUEUE_NAME REDIS_QUEUE_NAME" This reverts commit 1f83b4fc9b9dabda186257b38c265fe7012f90df. Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> Co-authored-by: Lauris BH Co-authored-by: techknowlogick Co-authored-by: Lunny Xiao --- custom/conf/app.ini.sample | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) (limited to 'custom') diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index 8b72ead3d6..29e147add8 100644 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -382,6 +382,39 @@ REPO_INDEXER_INCLUDE = ; A comma separated list of glob patterns to exclude from the index; ; default is empty REPO_INDEXER_EXCLUDE = +[queue] +; Specific queues can be individually configured with [queue.name]. [queue] provides defaults +; +; General queue queue type, currently support: persistable-channel, channel, level, redis, dummy +; default to persistable-channel +TYPE = persistable-channel +; data-dir for storing persistable queues and level queues, individual queues will be named by their type +DATADIR = queues/ +; Default queue length before a channel queue will block +LENGTH = 20 +; Batch size to send for batched queues +BATCH_LENGTH = 20 +; Connection string for redis queues this will store the redis connection string. +CONN_STR = "addrs=127.0.0.1:6379 db=0" +; Provide the suffix of the default redis queue name - specific queues can be overriden within in their [queue.name] sections. +QUEUE_NAME = "_queue" +; If the queue cannot be created at startup - level queues may need a timeout at startup - wrap the queue: +WRAP_IF_NECESSARY = true +; Attempt to create the wrapped queue at max +MAX_ATTEMPTS = 10 +; Timeout queue creation +TIMEOUT = 15m30s +; Create a pool with this many workers +WORKERS = 1 +; Dynamically scale the worker pool to at this many workers +MAX_WORKERS = 10 +; Add boost workers when the queue blocks for BLOCK_TIMEOUT +BLOCK_TIMEOUT = 1s +; Remove the boost workers after BOOST_TIMEOUT +BOOST_TIMEOUT = 5m +; During a boost add BOOST_WORKERS +BOOST_WORKERS = 5 + [admin] ; Disallow regular (non-admin) users from creating organizations. DISABLE_REGULAR_ORG_CREATION = false -- cgit v1.2.3