From c9c0773f2c2363dcfa37e675b83ec3f0b49bd0d9 Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Fri, 26 Apr 2024 13:50:46 +0100 Subject: [performance] update remaining worker pools to use queues (#2865) * start replacing client + federator + media workers with new worker + queue types * refactor federatingDB.Delete(), drop queued messages when deleting account / status * move all queue purging to the processor workers * undo toolchain updates * code comments, ensure dereferencer worker pool gets started * update gruf libraries in readme * start the job scheduler separately to the worker pools * reshuffle ordering or server.go + remove duplicate worker start / stop * update go-list version * fix vendoring * move queue invalidation to before wipeing / deletion, to ensure queued work not dropped * add logging to worker processing functions in testrig, don't start workers in unexpected places * update go-structr to add (+then rely on) QueueCtx{} type * ensure more worker pools get started properly in tests * fix remaining broken tests relying on worker queue logic * fix account test suite queue popping logic, ensure noop workers do not pull from queue * move back accidentally shuffled account deletion order * ensure error (non nil!!) gets passed in refactored federatingDB{}.Delete() * silently drop deletes from accounts not permitted to * don't warn log on forwarded deletes * make if else clauses easier to parse * use getFederatorMsg() * improved code comment * improved code comment re: requesting account delete checks * remove boolean result from worker start / stop since false = already running or already stopped * remove optional passed-in http.client * remove worker starting from the admin CLI commands (we don't need to handle side-effects) * update prune cli to start scheduler but not all of the workers * fix rebase issues * remove redundant return statements * i'm sorry sir linter --- internal/util/fns.go | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 internal/util/fns.go (limited to 'internal/util') diff --git a/internal/util/fns.go b/internal/util/fns.go new file mode 100644 index 000000000..53f5ebeed --- /dev/null +++ b/internal/util/fns.go @@ -0,0 +1,68 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package util + +import ( + "fmt" + "os" + "runtime" + + "codeberg.org/gruf/go-errors/v2" +) + +// Must executes 'fn' repeatedly until +// it successfully returns without panic. +func Must(fn func()) { + if fn == nil { + panic("nil func") + } + for !func() (done bool) { + defer func() { + if r := recover(); r != nil { + // Gather calling func frames. + pcs := make([]uintptr, 10) + n := runtime.Callers(3, pcs) + i := runtime.CallersFrames(pcs[:n]) + c := gatherFrames(i, n) + + const msg = "recovered panic: %v\n\n%s\n" + fmt.Fprintf(os.Stderr, msg, r, c.String()) + } + }() + fn() + done = true + return + }() { //nolint + } +} + +// gatherFrames collates runtime frames from a frame iterator. +func gatherFrames(iter *runtime.Frames, n int) errors.Callers { + if iter == nil { + return nil + } + frames := make([]runtime.Frame, 0, n) + for { + f, ok := iter.Next() + if !ok { + break + } + frames = append(frames, f) + } + return frames +} -- cgit v1.2.3