[feature] status refetch support (#1690)

* revamp http client to not limit requests, instead use sender worker

Signed-off-by: kim <grufwub@gmail.com>

* remove separate sender worker pool, spawn 2*GOMAXPROCS batch senders each time, no need for transport cache sweeping

Signed-off-by: kim <grufwub@gmail.com>

* improve batch senders to keep popping recipients until remote URL found

Signed-off-by: kim <grufwub@gmail.com>

* fix recipient looping issue

Signed-off-by: kim <grufwub@gmail.com>

* move request id ctx key to gtscontext, finish filling out more code comments, add basic support for not logging client IP

Signed-off-by: kim <grufwub@gmail.com>

* first draft of status refetching logic

Signed-off-by: kim <grufwub@gmail.com>

* fix testrig to use new federation alloc func signature

Signed-off-by: kim <grufwub@gmail.com>

* fix log format directive

Signed-off-by: kim <grufwub@gmail.com>

* add status fetched_at migration

Signed-off-by: kim <grufwub@gmail.com>

* remove unused / unchecked for error types

Signed-off-by: kim <grufwub@gmail.com>

* add back the used type...

Signed-off-by: kim <grufwub@gmail.com>

* add separate internal getStatus() function for derefThread() that doesn't recurse

Signed-off-by: kim <grufwub@gmail.com>

* improved mention and media attachment error handling

Signed-off-by: kim <grufwub@gmail.com>

* fix log and error format directives

Signed-off-by: kim <grufwub@gmail.com>

* update account deref to match status deref changes

Signed-off-by: kim <grufwub@gmail.com>

* very small code formatting change to make things clearer

Signed-off-by: kim <grufwub@gmail.com>

* add more code comments

Signed-off-by: kim <grufwub@gmail.com>

* improved code commenting

Signed-off-by: kim <grufwub@gmail.com>

* only check for required further derefs if needed

Signed-off-by: kim <grufwub@gmail.com>

* improved cache invalidation

Signed-off-by: kim <grufwub@gmail.com>

* tweak cache restarting to use a (very small) backoff

Signed-off-by: kim <grufwub@gmail.com>

* small readability changes and fixes

Signed-off-by: kim <grufwub@gmail.com>

* fix account sync issues

Signed-off-by: kim <grufwub@gmail.com>

* fix merge conflicts + update account enrichment to accept already-passed accountable

Signed-off-by: kim <grufwub@gmail.com>

* remove secondary function declaration

Signed-off-by: kim <grufwub@gmail.com>

* normalise dereferencer get status / account behaviour, fix remaining tests

Signed-off-by: kim <grufwub@gmail.com>

* fix remaining rebase conflicts, finish commenting code

Signed-off-by: kim <grufwub@gmail.com>

* appease the linter

Signed-off-by: kim <grufwub@gmail.com>

* add source file header

Signed-off-by: kim <grufwub@gmail.com>

* update to use TIMESTAMPTZ column type instead of just TIMESTAMP

Signed-off-by: kim <grufwub@gmail.com>

* don't pass in 'updated_at' to UpdateEmoji()

Signed-off-by: kim <grufwub@gmail.com>

* use new ap.Resolve{Account,Status}able() functions

Signed-off-by: kim <grufwub@gmail.com>

* remove the somewhat confusing rescoping of the same variable names

Signed-off-by: kim <grufwub@gmail.com>

* update migration file name, improved database delete error returns

Signed-off-by: kim <grufwub@gmail.com>

* formatting

Signed-off-by: kim <grufwub@gmail.com>

* improved multi-delete database functions to minimise DB calls

Signed-off-by: kim <grufwub@gmail.com>

* remove unused type

Signed-off-by: kim <grufwub@gmail.com>

* fix delete statements

Signed-off-by: kim <grufwub@gmail.com>

---------

Signed-off-by: kim <grufwub@gmail.com>

1 files changed, 0 insertions, 66 deletions

diff --git a/internal/federation/dereferencing/error.go b/internal/federation/dereferencing/error.go
index 769150cef..1b8d90653 100644
--- a/internal/federation/dereferencing/error.go
+++ b/internal/federation/dereferencing/error.go
@@ -19,25 +19,8 @@ package dereferencing
 
 import (
 	"fmt"
-	"net/http"
-
-	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 )
 
-// ErrDB denotes that a proper error has occurred when doing
-// a database call, as opposed to a simple db.ErrNoEntries.
-type ErrDB struct {
-	wrapped error
-}
-
-func (err *ErrDB) Error() string {
-	return fmt.Sprintf("database error during dereferencing: %v", err.wrapped)
-}
-
-func newErrDB(err error) error {
-	return &ErrDB{wrapped: err}
-}
-
 // ErrNotRetrievable denotes that an item could not be dereferenced
 // with the given parameters.
 type ErrNotRetrievable struct {
@@ -51,52 +34,3 @@ func (err *ErrNotRetrievable) Error() string {
 func NewErrNotRetrievable(err error) error {
 	return &ErrNotRetrievable{wrapped: err}
 }
-
-// ErrTransportError indicates that something unforeseen went wrong creating
-// a transport, or while making an http call to a remote resource with a transport.
-type ErrTransportError struct {
-	wrapped error
-}
-
-func (err *ErrTransportError) Error() string {
-	return fmt.Sprintf("transport error: %v", err.wrapped)
-}
-
-func newErrTransportError(err error) error {
-	return &ErrTransportError{wrapped: err}
-}
-
-// ErrOther denotes some other kind of weird error, perhaps from a malformed json
-// or some other weird crapola.
-type ErrOther struct {
-	wrapped error
-}
-
-func (err *ErrOther) Error() string {
-	return fmt.Sprintf("unexpected error: %v", err.wrapped)
-}
-
-func newErrOther(err error) error {
-	return &ErrOther{wrapped: err}
-}
-
-func wrapDerefError(derefErr error, fluff string) error {
-	// Wrap with fluff.
-	err := derefErr
-	if fluff != "" {
-		err = fmt.Errorf("%s: %w", fluff, derefErr)
-	}
-
-	// Check for unretrievable HTTP status code errors.
-	if code := gtserror.StatusCode(derefErr); // nocollapse
-	code == http.StatusGone || code == http.StatusNotFound {
-		return NewErrNotRetrievable(err)
-	}
-
-	// Check for other untrievable errors.
-	if gtserror.NotFound(derefErr) {
-		return NewErrNotRetrievable(err)
-	}
-
-	return err
-}