From 0e572460830ce7767562f08034f96d51e41b6349 Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Tue, 8 Nov 2022 09:35:24 +0000 Subject: [feature] various worker / request queue improvements (#995) * greatly simplify httpclient request queuing Signed-off-by: kim * improved request queue mutex logic Signed-off-by: kim * use improved hashmap library Signed-off-by: kim * add warn logging when request queues are full Signed-off-by: kim * improve worker pool prefix var naming Signed-off-by: kim * improved worker pool error logging Signed-off-by: kim * move error message into separate field Signed-off-by: kim * remove old log statement Signed-off-by: kim * don't export worker message, it gets very spammy :') Signed-off-by: kim Signed-off-by: kim --- internal/httpclient/queue.go | 68 -------------------------------------------- 1 file changed, 68 deletions(-) delete mode 100644 internal/httpclient/queue.go (limited to 'internal/httpclient/queue.go') diff --git a/internal/httpclient/queue.go b/internal/httpclient/queue.go deleted file mode 100644 index 8cb1274be..000000000 --- a/internal/httpclient/queue.go +++ /dev/null @@ -1,68 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -package httpclient - -import ( - "strings" - "sync" - - "github.com/superseriousbusiness/gotosocial/internal/log" -) - -type requestQueue struct { - hostQueues sync.Map // map of `hostQueue` - maxOpenConns int // max open conns per host per request method -} - -type hostQueue struct { - slotsByMethod sync.Map -} - -// getWaitSpot returns a wait channel and release function for http clients -// that want to do requests politely: that is, wait for their turn. -// -// To wait, a caller should do a select on an attempted insert into the -// returned wait channel. Once the insert succeeds, then the caller should -// proceed with the http request that pertains to the given host + method. -// It doesn't matter what's put into the wait channel, just any interface{}. -// -// When the caller is finished with their http request, they should free up the -// slot they were occupying in the wait queue, by calling the release function. -// -// The reason for the caller needing to provide host and method, is that each -// remote host has a separate wait queue, and there's a separate wait queue -// per method for that host as well. This ensures that outgoing requests can still -// proceed for others hosts and methods while other requests are undergoing, -// while also preventing one host from being spammed with, for example, a -// shitload of GET requests all at once. -func (rc *requestQueue) getWaitSpot(host string, method string) (wait chan<- interface{}, release func()) { - hostQueueI, _ := rc.hostQueues.LoadOrStore(host, new(hostQueue)) - hostQueue, ok := hostQueueI.(*hostQueue) - if !ok { - log.Panic("hostQueueI was not a *hostQueue") - } - - waitSlotI, _ := hostQueue.slotsByMethod.LoadOrStore(strings.ToUpper(method), make(chan interface{}, rc.maxOpenConns)) - methodQueue, ok := waitSlotI.(chan interface{}) - if !ok { - log.Panic("waitSlotI was not a chan interface{}") - } - - return methodQueue, func() { <-methodQueue } -} -- cgit v1.2.3