[performance] update remaining worker pools to use queues (#2865)

* start replacing client + federator + media workers with new worker + queue types

* refactor federatingDB.Delete(), drop queued messages when deleting account / status

* move all queue purging to the processor workers

* undo toolchain updates

* code comments, ensure dereferencer worker pool gets started

* update gruf libraries in readme

* start the job scheduler separately to the worker pools

* reshuffle ordering or server.go + remove duplicate worker start / stop

* update go-list version

* fix vendoring

* move queue invalidation to before wipeing / deletion, to ensure queued work not dropped

* add logging to worker processing functions in testrig, don't start workers in unexpected places

* update go-structr to add (+then rely on) QueueCtx{} type

* ensure more worker pools get started properly in tests

* fix remaining broken tests relying on worker queue logic

* fix account test suite queue popping logic, ensure noop workers do not pull from queue

* move back accidentally shuffled account deletion order

* ensure error (non nil!!) gets passed in refactored federatingDB{}.Delete()

* silently drop deletes from accounts not permitted to

* don't warn log on forwarded deletes

* make if else clauses easier to parse

* use getFederatorMsg()

* improved code comment

* improved code comment re: requesting account delete checks

* remove boolean result from worker start / stop since false = already running or already stopped

* remove optional passed-in http.client

* remove worker starting from the admin CLI commands (we don't need to handle side-effects)

* update prune cli to start scheduler but not all of the workers

* fix rebase issues

* remove redundant return statements

* i'm sorry sir linter

1 files changed, 204 insertions, 0 deletions

diff --git a/vendor/codeberg.org/gruf/go-list/list.go b/vendor/codeberg.org/gruf/go-list/list.go
new file mode 100644
index 000000000..5490fa636
--- /dev/null
+++ b/vendor/codeberg.org/gruf/go-list/list.go
@@ -0,0 +1,204 @@
+package list
+
+// Elem represents an element in a doubly-linked list.
+type Elem[T any] struct {
+	Next  *Elem[T]
+	Prev  *Elem[T]
+	Value T
+}
+
+// List implements a doubly-linked list, where:
+// - Head = index 0   (i.e. the front)
+// - Tail = index n-1 (i.e. the back)
+type List[T any] struct {
+	Head *Elem[T]
+	Tail *Elem[T]
+	len  int
+}
+
+// Len returns the current list length.
+func (l *List[T]) Len() int {
+	return l.len
+}
+
+// PushFront adds 'v' to the beginning of the list.
+func (l *List[T]) PushFront(v T) *Elem[T] {
+	elem := &Elem[T]{Value: v}
+	l.PushElemFront(elem)
+	return elem
+}
+
+// PushBack adds 'v' to the end of the list.
+func (l *List[T]) PushBack(v T) *Elem[T] {
+	elem := &Elem[T]{Value: v}
+	l.PushElemBack(elem)
+	return elem
+}
+
+// InsertBefore adds 'v' into the list before 'at'.
+func (l *List[T]) InsertBefore(v T, at *Elem[T]) *Elem[T] {
+	elem := &Elem[T]{Value: v}
+	l.InsertElemBefore(elem, at)
+	return elem
+}
+
+// InsertAfter adds 'v' into the list after 'at'.
+func (l *List[T]) InsertAfter(v T, at *Elem[T]) *Elem[T] {
+	elem := &Elem[T]{Value: v}
+	l.InsertElemAfter(elem, at)
+	return elem
+}
+
+// PushFrontNode adds 'elem' to the front of the list.
+func (l *List[T]) PushElemFront(elem *Elem[T]) {
+	if elem == l.Head {
+		return
+	}
+
+	// Set new head.
+	oldHead := l.Head
+	l.Head = elem
+
+	if oldHead != nil {
+		// Link to old head
+		elem.Next = oldHead
+		oldHead.Prev = elem
+	} else {
+		// First in list.
+		l.Tail = elem
+	}
+
+	// Incr count
+	l.len++
+}
+
+// PushBackNode adds 'elem' to the back of the list.
+func (l *List[T]) PushElemBack(elem *Elem[T]) {
+	if elem == l.Tail {
+		return
+	}
+
+	// Set new tail.
+	oldTail := l.Tail
+	l.Tail = elem
+
+	if oldTail != nil {
+		// Link to old tail
+		elem.Prev = oldTail
+		oldTail.Next = elem
+	} else {
+		// First in list.
+		l.Head = elem
+	}
+
+	// Incr count
+	l.len++
+}
+
+// InsertElemAfter adds 'elem' into the list after 'at' (i.e. at.Next = elem).
+func (l *List[T]) InsertElemAfter(elem *Elem[T], at *Elem[T]) {
+	if elem == at {
+		return
+	}
+
+	// Set new 'next'.
+	oldNext := at.Next
+	at.Next = elem
+
+	// Link to 'at'.
+	elem.Prev = at
+
+	if oldNext == nil {
+		// Set new tail
+		l.Tail = elem
+	} else {
+		// Link to 'prev'.
+		oldNext.Prev = elem
+		elem.Next = oldNext
+	}
+
+	// Incr count
+	l.len++
+}
+
+// InsertElemBefore adds 'elem' into the list before 'at' (i.e. at.Prev = elem).
+func (l *List[T]) InsertElemBefore(elem *Elem[T], at *Elem[T]) {
+	if elem == at {
+		return
+	}
+
+	// Set new 'prev'.
+	oldPrev := at.Prev
+	at.Prev = elem
+
+	// Link to 'at'.
+	elem.Next = at
+
+	if oldPrev == nil {
+		// Set new head
+		l.Head = elem
+	} else {
+		// Link to 'next'.
+		oldPrev.Next = elem
+		elem.Prev = oldPrev
+	}
+
+	// Incr count
+	l.len++
+}
+
+// Remove removes the 'elem' from the list.
+func (l *List[T]) Remove(elem *Elem[T]) {
+	// Get linked elems.
+	next := elem.Next
+	prev := elem.Prev
+
+	// Unset elem.
+	elem.Next = nil
+	elem.Prev = nil
+
+	switch {
+	// elem is ONLY one in list.
+	case next == nil && prev == nil:
+		l.Head = nil
+		l.Tail = nil
+
+	// elem is front in list.
+	case next != nil && prev == nil:
+		l.Head = next
+		next.Prev = nil
+
+	// elem is last in list.
+	case prev != nil && next == nil:
+		l.Tail = prev
+		prev.Next = nil
+
+	// elem in middle of list.
+	default:
+		next.Prev = prev
+		prev.Next = next
+	}
+
+	// Decr count
+	l.len--
+}
+
+// Range calls 'fn' on every element from head forward in list.
+func (l *List[T]) Range(fn func(*Elem[T])) {
+	if fn == nil {
+		panic("nil function")
+	}
+	for elem := l.Head; elem != nil; elem = elem.Next {
+		fn(elem)
+	}
+}
+
+// RangeReverse calls 'fn' on every element from tail backward in list.
+func (l *List[T]) RangeReverse(fn func(*Elem[T])) {
+	if fn == nil {
+		panic("nil function")
+	}
+	for elem := l.Tail; elem != nil; elem = elem.Prev {
+		fn(elem)
+	}
+}