You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

queue.go 3.4KB

Graceful Queues: Issue Indexing and Tasks (#9363) * Queue: Add generic graceful queues with settings * Queue & Setting: Add worker pool implementation * Queue: Add worker settings * Queue: Make resizing worker pools * Queue: Add name variable to queues * Queue: Add monitoring * Queue: Improve logging * Issues: Gracefulise the issues indexer Remove the old now unused specific queues * Task: Move to generic queue and gracefulise * Issues: Standardise the issues indexer queue settings * Fix test * Queue: Allow Redis to connect to unix * Prevent deadlock during early shutdown of issue indexer * Add MaxWorker settings to queues * Merge branch 'master' into graceful-queues * Update modules/indexer/issues/indexer.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update modules/indexer/issues/indexer.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update modules/queue/queue_channel.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update modules/queue/queue_disk.go * Update modules/queue/queue_disk_channel.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Rename queue.Description to queue.ManagedQueue as per @guillep2k * Cancel pool workers when removed * Remove dependency on queue from setting * Update modules/queue/queue_redis.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * As per @guillep2k add mutex locks on shutdown/terminate * move unlocking out of setInternal * Add warning if number of workers < 0 * Small changes as per @guillep2k * No redis host specified not found * Clean up documentation for queues * Update docs/content/doc/advanced/config-cheat-sheet.en-us.md * Update modules/indexer/issues/indexer_test.go * Ensure that persistable channel queue is added to manager * Rename QUEUE_NAME REDIS_QUEUE_NAME * Revert "Rename QUEUE_NAME REDIS_QUEUE_NAME" This reverts commit 1f83b4fc9b9dabda186257b38c265fe7012f90df. Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: techknowlogick <matti@mdranta.net> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
4 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133
  1. // Copyright 2019 The Gitea Authors. All rights reserved.
  2. // Use of this source code is governed by a MIT-style
  3. // license that can be found in the LICENSE file.
  4. package queue
  5. import (
  6. "context"
  7. "encoding/json"
  8. "fmt"
  9. "reflect"
  10. )
  11. // ErrInvalidConfiguration is called when there is invalid configuration for a queue
  12. type ErrInvalidConfiguration struct {
  13. cfg interface{}
  14. err error
  15. }
  16. func (err ErrInvalidConfiguration) Error() string {
  17. if err.err != nil {
  18. return fmt.Sprintf("Invalid Configuration Argument: %v: Error: %v", err.cfg, err.err)
  19. }
  20. return fmt.Sprintf("Invalid Configuration Argument: %v", err.cfg)
  21. }
  22. // IsErrInvalidConfiguration checks if an error is an ErrInvalidConfiguration
  23. func IsErrInvalidConfiguration(err error) bool {
  24. _, ok := err.(ErrInvalidConfiguration)
  25. return ok
  26. }
  27. // Type is a type of Queue
  28. type Type string
  29. // Data defines an type of queuable data
  30. type Data interface{}
  31. // HandlerFunc is a function that takes a variable amount of data and processes it
  32. type HandlerFunc func(...Data)
  33. // NewQueueFunc is a function that creates a queue
  34. type NewQueueFunc func(handler HandlerFunc, config interface{}, exemplar interface{}) (Queue, error)
  35. // Shutdownable represents a queue that can be shutdown
  36. type Shutdownable interface {
  37. Shutdown()
  38. Terminate()
  39. }
  40. // Named represents a queue with a name
  41. type Named interface {
  42. Name() string
  43. }
  44. // Queue defines an interface to save an issue indexer queue
  45. type Queue interface {
  46. Run(atShutdown, atTerminate func(context.Context, func()))
  47. Push(Data) error
  48. }
  49. // DummyQueueType is the type for the dummy queue
  50. const DummyQueueType Type = "dummy"
  51. // NewDummyQueue creates a new DummyQueue
  52. func NewDummyQueue(handler HandlerFunc, opts, exemplar interface{}) (Queue, error) {
  53. return &DummyQueue{}, nil
  54. }
  55. // DummyQueue represents an empty queue
  56. type DummyQueue struct {
  57. }
  58. // Run starts to run the queue
  59. func (b *DummyQueue) Run(_, _ func(context.Context, func())) {}
  60. // Push pushes data to the queue
  61. func (b *DummyQueue) Push(Data) error {
  62. return nil
  63. }
  64. func toConfig(exemplar, cfg interface{}) (interface{}, error) {
  65. if reflect.TypeOf(cfg).AssignableTo(reflect.TypeOf(exemplar)) {
  66. return cfg, nil
  67. }
  68. configBytes, ok := cfg.([]byte)
  69. if !ok {
  70. configStr, ok := cfg.(string)
  71. if !ok {
  72. return nil, ErrInvalidConfiguration{cfg: cfg}
  73. }
  74. configBytes = []byte(configStr)
  75. }
  76. newVal := reflect.New(reflect.TypeOf(exemplar))
  77. if err := json.Unmarshal(configBytes, newVal.Interface()); err != nil {
  78. return nil, ErrInvalidConfiguration{cfg: cfg, err: err}
  79. }
  80. return newVal.Elem().Interface(), nil
  81. }
  82. var queuesMap = map[Type]NewQueueFunc{DummyQueueType: NewDummyQueue}
  83. // RegisteredTypes provides the list of requested types of queues
  84. func RegisteredTypes() []Type {
  85. types := make([]Type, len(queuesMap))
  86. i := 0
  87. for key := range queuesMap {
  88. types[i] = key
  89. i++
  90. }
  91. return types
  92. }
  93. // RegisteredTypesAsString provides the list of requested types of queues
  94. func RegisteredTypesAsString() []string {
  95. types := make([]string, len(queuesMap))
  96. i := 0
  97. for key := range queuesMap {
  98. types[i] = string(key)
  99. i++
  100. }
  101. return types
  102. }
  103. // NewQueue takes a queue Type and HandlerFunc some options and possibly an exemplar and returns a Queue or an error
  104. func NewQueue(queueType Type, handlerFunc HandlerFunc, opts, exemplar interface{}) (Queue, error) {
  105. newFn, ok := queuesMap[queueType]
  106. if !ok {
  107. return nil, fmt.Errorf("Unsupported queue type: %v", queueType)
  108. }
  109. return newFn(handlerFunc, opts, exemplar)
  110. }