aboutsummaryrefslogtreecommitdiffstats
path: root/modules/queue/unique_queue_channel.go
diff options
context:
space:
mode:
Diffstat (limited to 'modules/queue/unique_queue_channel.go')
-rw-r--r--modules/queue/unique_queue_channel.go212
1 files changed, 0 insertions, 212 deletions
diff --git a/modules/queue/unique_queue_channel.go b/modules/queue/unique_queue_channel.go
deleted file mode 100644
index 62c051aa39..0000000000
--- a/modules/queue/unique_queue_channel.go
+++ /dev/null
@@ -1,212 +0,0 @@
-// Copyright 2020 The Gitea Authors. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-package queue
-
-import (
- "context"
- "fmt"
- "runtime/pprof"
- "sync"
- "time"
-
- "code.gitea.io/gitea/modules/container"
- "code.gitea.io/gitea/modules/json"
- "code.gitea.io/gitea/modules/log"
-)
-
-// ChannelUniqueQueueType is the type for channel queue
-const ChannelUniqueQueueType Type = "unique-channel"
-
-// ChannelUniqueQueueConfiguration is the configuration for a ChannelUniqueQueue
-type ChannelUniqueQueueConfiguration ChannelQueueConfiguration
-
-// ChannelUniqueQueue implements UniqueQueue
-//
-// It is basically a thin wrapper around a WorkerPool but keeps a store of
-// what has been pushed within a table.
-//
-// Please note that this Queue does not guarantee that a particular
-// task cannot be processed twice or more at the same time. Uniqueness is
-// only guaranteed whilst the task is waiting in the queue.
-type ChannelUniqueQueue struct {
- *WorkerPool
- lock sync.Mutex
- table container.Set[string]
- shutdownCtx context.Context
- shutdownCtxCancel context.CancelFunc
- terminateCtx context.Context
- terminateCtxCancel context.CancelFunc
- exemplar interface{}
- workers int
- name string
-}
-
-// NewChannelUniqueQueue create a memory channel queue
-func NewChannelUniqueQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) {
- configInterface, err := toConfig(ChannelUniqueQueueConfiguration{}, cfg)
- if err != nil {
- return nil, err
- }
- config := configInterface.(ChannelUniqueQueueConfiguration)
- if config.BatchLength == 0 {
- config.BatchLength = 1
- }
-
- terminateCtx, terminateCtxCancel := context.WithCancel(context.Background())
- shutdownCtx, shutdownCtxCancel := context.WithCancel(terminateCtx)
-
- queue := &ChannelUniqueQueue{
- table: make(container.Set[string]),
- shutdownCtx: shutdownCtx,
- shutdownCtxCancel: shutdownCtxCancel,
- terminateCtx: terminateCtx,
- terminateCtxCancel: terminateCtxCancel,
- exemplar: exemplar,
- workers: config.Workers,
- name: config.Name,
- }
- queue.WorkerPool = NewWorkerPool(func(data ...Data) (unhandled []Data) {
- for _, datum := range data {
- // No error is possible here because PushFunc ensures that this can be marshalled
- bs, _ := json.Marshal(datum)
-
- queue.lock.Lock()
- queue.table.Remove(string(bs))
- queue.lock.Unlock()
-
- if u := handle(datum); u != nil {
- if queue.IsPaused() {
- // We can only pushback to the channel if we're paused.
- go func() {
- if err := queue.Push(u[0]); err != nil {
- log.Error("Unable to push back to queue %d. Error: %v", queue.qid, err)
- }
- }()
- } else {
- unhandled = append(unhandled, u...)
- }
- }
- }
- return unhandled
- }, config.WorkerPoolConfiguration)
-
- queue.qid = GetManager().Add(queue, ChannelUniqueQueueType, config, exemplar)
- return queue, nil
-}
-
-// Run starts to run the queue
-func (q *ChannelUniqueQueue) Run(atShutdown, atTerminate func(func())) {
- pprof.SetGoroutineLabels(q.baseCtx)
- atShutdown(q.Shutdown)
- atTerminate(q.Terminate)
- log.Debug("ChannelUniqueQueue: %s Starting", q.name)
- _ = q.AddWorkers(q.workers, 0)
-}
-
-// Push will push data into the queue if the data is not already in the queue
-func (q *ChannelUniqueQueue) Push(data Data) error {
- return q.PushFunc(data, nil)
-}
-
-// PushFunc will push data into the queue
-func (q *ChannelUniqueQueue) PushFunc(data Data, fn func() error) error {
- if !assignableTo(data, q.exemplar) {
- return fmt.Errorf("unable to assign data: %v to same type as exemplar: %v in queue: %s", data, q.exemplar, q.name)
- }
-
- bs, err := json.Marshal(data)
- if err != nil {
- return err
- }
- q.lock.Lock()
- locked := true
- defer func() {
- if locked {
- q.lock.Unlock()
- }
- }()
- if !q.table.Add(string(bs)) {
- return ErrAlreadyInQueue
- }
- // FIXME: We probably need to implement some sort of limit here
- // If the downstream queue blocks this table will grow without limit
- if fn != nil {
- err := fn()
- if err != nil {
- q.table.Remove(string(bs))
- return err
- }
- }
- locked = false
- q.lock.Unlock()
- q.WorkerPool.Push(data)
- return nil
-}
-
-// Has checks if the data is in the queue
-func (q *ChannelUniqueQueue) Has(data Data) (bool, error) {
- bs, err := json.Marshal(data)
- if err != nil {
- return false, err
- }
-
- q.lock.Lock()
- defer q.lock.Unlock()
- return q.table.Contains(string(bs)), nil
-}
-
-// Flush flushes the channel with a timeout - the Flush worker will be registered as a flush worker with the manager
-func (q *ChannelUniqueQueue) Flush(timeout time.Duration) error {
- if q.IsPaused() {
- return nil
- }
- ctx, cancel := q.commonRegisterWorkers(1, timeout, true)
- defer cancel()
- return q.FlushWithContext(ctx)
-}
-
-// Shutdown processing from this queue
-func (q *ChannelUniqueQueue) Shutdown() {
- log.Trace("ChannelUniqueQueue: %s Shutting down", q.name)
- select {
- case <-q.shutdownCtx.Done():
- return
- default:
- }
- go func() {
- log.Trace("ChannelUniqueQueue: %s Flushing", q.name)
- if err := q.FlushWithContext(q.terminateCtx); err != nil {
- if !q.IsEmpty() {
- log.Warn("ChannelUniqueQueue: %s Terminated before completed flushing", q.name)
- }
- return
- }
- log.Debug("ChannelUniqueQueue: %s Flushed", q.name)
- }()
- q.shutdownCtxCancel()
- log.Debug("ChannelUniqueQueue: %s Shutdown", q.name)
-}
-
-// Terminate this queue and close the queue
-func (q *ChannelUniqueQueue) Terminate() {
- log.Trace("ChannelUniqueQueue: %s Terminating", q.name)
- q.Shutdown()
- select {
- case <-q.terminateCtx.Done():
- return
- default:
- }
- q.terminateCtxCancel()
- q.baseCtxFinished()
- log.Debug("ChannelUniqueQueue: %s Terminated", q.name)
-}
-
-// Name returns the name of this queue
-func (q *ChannelUniqueQueue) Name() string {
- return q.name
-}
-
-func init() {
- queuesMap[ChannelUniqueQueueType] = NewChannelUniqueQueue
-}