summaryrefslogtreecommitdiff
path: root/internal/federation
diff options
context:
space:
mode:
Diffstat (limited to 'internal/federation')
-rw-r--r--internal/federation/dereferencing/account.go184
-rw-r--r--internal/federation/dereferencing/emoji.go383
-rw-r--r--internal/federation/dereferencing/emoji_test.go45
-rw-r--r--internal/federation/dereferencing/media.go215
-rw-r--r--internal/federation/dereferencing/status.go241
-rw-r--r--internal/federation/dereferencing/util.go124
6 files changed, 712 insertions, 480 deletions
diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go
index 069fca1bc..e48507124 100644
--- a/internal/federation/dereferencing/account.go
+++ b/internal/federation/dereferencing/account.go
@@ -33,7 +33,6 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/media"
- "github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
@@ -730,18 +729,18 @@ func (d *Dereferencer) enrichAccount(
latestAcc.ID = account.ID
latestAcc.FetchedAt = time.Now()
- // Ensure the account's avatar media is populated, passing in existing to check for changes.
- if err := d.fetchRemoteAccountAvatar(ctx, tsport, account, latestAcc); err != nil {
+ // Ensure the account's avatar media is populated, passing in existing to check for chages.
+ if err := d.fetchAccountAvatar(ctx, requestUser, account, latestAcc); err != nil {
log.Errorf(ctx, "error fetching remote avatar for account %s: %v", uri, err)
}
- // Ensure the account's avatar media is populated, passing in existing to check for changes.
- if err := d.fetchRemoteAccountHeader(ctx, tsport, account, latestAcc); err != nil {
+ // Ensure the account's avatar media is populated, passing in existing to check for chages.
+ if err := d.fetchAccountHeader(ctx, requestUser, account, latestAcc); err != nil {
log.Errorf(ctx, "error fetching remote header for account %s: %v", uri, err)
}
// Fetch the latest remote account emoji IDs used in account display name/bio.
- if _, err = d.fetchRemoteAccountEmojis(ctx, latestAcc, requestUser); err != nil {
+ if err = d.fetchAccountEmojis(ctx, account, latestAcc); err != nil {
log.Errorf(ctx, "error fetching remote emojis for account %s: %v", uri, err)
}
@@ -779,9 +778,9 @@ func (d *Dereferencer) enrichAccount(
return latestAcc, apubAcc, nil
}
-func (d *Dereferencer) fetchRemoteAccountAvatar(
+func (d *Dereferencer) fetchAccountAvatar(
ctx context.Context,
- tsport transport.Transport,
+ requestUser string,
existingAcc *gtsmodel.Account,
latestAcc *gtsmodel.Account,
) error {
@@ -808,7 +807,7 @@ func (d *Dereferencer) fetchRemoteAccountAvatar(
// Ensuring existing attachment is up-to-date
// and any recaching is performed if required.
existing, err := d.updateAttachment(ctx,
- tsport,
+ requestUser,
existing,
nil,
)
@@ -830,18 +829,23 @@ func (d *Dereferencer) fetchRemoteAccountAvatar(
}
}
- // Fetch newly changed avatar from remote.
- attachment, err := d.loadAttachment(ctx,
- tsport,
+ // Fetch newly changed avatar.
+ attachment, err := d.GetMedia(ctx,
+ requestUser,
latestAcc.ID,
latestAcc.AvatarRemoteURL,
- &media.AdditionalMediaInfo{
+ media.AdditionalMediaInfo{
Avatar: util.Ptr(true),
RemoteURL: &latestAcc.AvatarRemoteURL,
},
)
if err != nil {
- return gtserror.Newf("error loading attachment %s: %w", latestAcc.AvatarRemoteURL, err)
+ if attachment == nil {
+ return gtserror.Newf("error loading attachment %s: %w", latestAcc.AvatarRemoteURL, err)
+ }
+
+ // non-fatal error occurred during loading, still use it.
+ log.Warnf(ctx, "partially loaded attachment: %v", err)
}
// Set the avatar attachment on account model.
@@ -851,9 +855,9 @@ func (d *Dereferencer) fetchRemoteAccountAvatar(
return nil
}
-func (d *Dereferencer) fetchRemoteAccountHeader(
+func (d *Dereferencer) fetchAccountHeader(
ctx context.Context,
- tsport transport.Transport,
+ requestUser string,
existingAcc *gtsmodel.Account,
latestAcc *gtsmodel.Account,
) error {
@@ -880,7 +884,7 @@ func (d *Dereferencer) fetchRemoteAccountHeader(
// Ensuring existing attachment is up-to-date
// and any recaching is performed if required.
existing, err := d.updateAttachment(ctx,
- tsport,
+ requestUser,
existing,
nil,
)
@@ -902,18 +906,23 @@ func (d *Dereferencer) fetchRemoteAccountHeader(
}
}
- // Fetch newly changed header from remote.
- attachment, err := d.loadAttachment(ctx,
- tsport,
+ // Fetch newly changed header.
+ attachment, err := d.GetMedia(ctx,
+ requestUser,
latestAcc.ID,
latestAcc.HeaderRemoteURL,
- &media.AdditionalMediaInfo{
+ media.AdditionalMediaInfo{
Header: util.Ptr(true),
RemoteURL: &latestAcc.HeaderRemoteURL,
},
)
if err != nil {
- return gtserror.Newf("error loading attachment %s: %w", latestAcc.HeaderRemoteURL, err)
+ if attachment == nil {
+ return gtserror.Newf("error loading attachment %s: %w", latestAcc.HeaderRemoteURL, err)
+ }
+
+ // non-fatal error occurred during loading, still use it.
+ log.Warnf(ctx, "partially loaded attachment: %v", err)
}
// Set the header attachment on account model.
@@ -923,119 +932,44 @@ func (d *Dereferencer) fetchRemoteAccountHeader(
return nil
}
-func (d *Dereferencer) fetchRemoteAccountEmojis(ctx context.Context, targetAccount *gtsmodel.Account, requestingUsername string) (bool, error) {
- maybeEmojis := targetAccount.Emojis
- maybeEmojiIDs := targetAccount.EmojiIDs
-
- // It's possible that the account had emoji IDs set on it, but not Emojis
- // themselves, depending on how it was fetched before being passed to us.
- //
- // If we only have IDs, fetch the emojis from the db. We know they're in
- // there or else they wouldn't have IDs.
- if len(maybeEmojiIDs) > len(maybeEmojis) {
- maybeEmojis = make([]*gtsmodel.Emoji, 0, len(maybeEmojiIDs))
- for _, emojiID := range maybeEmojiIDs {
- maybeEmoji, err := d.state.DB.GetEmojiByID(ctx, emojiID)
- if err != nil {
- return false, err
- }
- maybeEmojis = append(maybeEmojis, maybeEmoji)
- }
- }
-
- // For all the maybe emojis we have, we either fetch them from the database
- // (if we haven't already), or dereference them from the remote instance.
- gotEmojis, err := d.populateEmojis(ctx, maybeEmojis, requestingUsername)
- if err != nil {
- return false, err
- }
-
- // Extract the ID of each fetched or dereferenced emoji, so we can attach
- // this to the account if necessary.
- gotEmojiIDs := make([]string, 0, len(gotEmojis))
- for _, e := range gotEmojis {
- gotEmojiIDs = append(gotEmojiIDs, e.ID)
- }
-
- var (
- changed = false // have the emojis for this account changed?
- maybeLen = len(maybeEmojis)
- gotLen = len(gotEmojis)
+func (d *Dereferencer) fetchAccountEmojis(
+ ctx context.Context,
+ existing *gtsmodel.Account,
+ account *gtsmodel.Account,
+) error {
+ // Fetch the updated emojis for our account.
+ emojis, changed, err := d.fetchEmojis(ctx,
+ existing.Emojis,
+ account.Emojis,
)
-
- // if the length of everything is zero, this is simple:
- // nothing has changed and there's nothing to do
- if maybeLen == 0 && gotLen == 0 {
- return changed, nil
- }
-
- // if the *amount* of emojis on the account has changed, then the got emojis
- // are definitely different from the previous ones (if there were any) --
- // the account has either more or fewer emojis set on it now, so take the
- // discovered emojis as the new correct ones.
- if maybeLen != gotLen {
- changed = true
- targetAccount.Emojis = gotEmojis
- targetAccount.EmojiIDs = gotEmojiIDs
- return changed, nil
+ if err != nil {
+ return gtserror.Newf("error fetching emojis: %w", err)
}
- // if the lengths are the same but not all of the slices are
- // zero, something *might* have changed, so we have to check
-
- // 1. did we have emojis before that we don't have now?
- for _, maybeEmoji := range maybeEmojis {
- var stillPresent bool
-
- for _, gotEmoji := range gotEmojis {
- if maybeEmoji.URI == gotEmoji.URI {
- // the emoji we maybe had is still present now,
- // so we can stop checking gotEmojis
- stillPresent = true
- break
- }
- }
-
- if !stillPresent {
- // at least one maybeEmoji is no longer present in
- // the got emojis, so we can stop checking now
- changed = true
- targetAccount.Emojis = gotEmojis
- targetAccount.EmojiIDs = gotEmojiIDs
- return changed, nil
- }
+ if !changed {
+ // Use existing account emoji objects.
+ account.EmojiIDs = existing.EmojiIDs
+ account.Emojis = existing.Emojis
+ return nil
}
- // 2. do we have emojis now that we didn't have before?
- for _, gotEmoji := range gotEmojis {
- var wasPresent bool
-
- for _, maybeEmoji := range maybeEmojis {
- // check emoji IDs here as well, because unreferenced
- // maybe emojis we didn't already have would not have
- // had IDs set on them yet
- if gotEmoji.URI == maybeEmoji.URI && gotEmoji.ID == maybeEmoji.ID {
- // this got emoji was present already in the maybeEmoji,
- // so we can stop checking through maybeEmojis
- wasPresent = true
- break
- }
- }
+ // Set latest emojis.
+ account.Emojis = emojis
- if !wasPresent {
- // at least one gotEmojis was not present in
- // the maybeEmojis, so we can stop checking now
- changed = true
- targetAccount.Emojis = gotEmojis
- targetAccount.EmojiIDs = gotEmojiIDs
- return changed, nil
- }
+ // Iterate over and set changed emoji IDs.
+ account.EmojiIDs = make([]string, len(emojis))
+ for i, emoji := range emojis {
+ account.EmojiIDs[i] = emoji.ID
}
- return changed, nil
+ return nil
}
-func (d *Dereferencer) dereferenceAccountStats(ctx context.Context, requestUser string, account *gtsmodel.Account) error {
+func (d *Dereferencer) dereferenceAccountStats(
+ ctx context.Context,
+ requestUser string,
+ account *gtsmodel.Account,
+) error {
// Ensure we have a stats model for this account.
if account.Stats == nil {
if err := d.state.DB.PopulateAccountStats(ctx, account); err != nil {
diff --git a/internal/federation/dereferencing/emoji.go b/internal/federation/dereferencing/emoji.go
index e81737d04..16f5acf25 100644
--- a/internal/federation/dereferencing/emoji.go
+++ b/internal/federation/dereferencing/emoji.go
@@ -19,29 +19,190 @@ package dereferencing
import (
"context"
- "fmt"
+ "errors"
"io"
"net/url"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
- "github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
-func (d *Dereferencer) GetRemoteEmoji(ctx context.Context, requestUser string, remoteURL string, shortcode string, domain string, id string, emojiURI string, ai *media.AdditionalEmojiInfo, refresh bool) (*media.ProcessingEmoji, error) {
- var shortcodeDomain = shortcode + "@" + domain
+// GetEmoji fetches the emoji with given shortcode,
+// domain and remote URL to dereference it by. This
+// handles the case of existing emojis by passing them
+// to RefreshEmoji(), which in the case of a local
+// emoji will be a no-op. If the emoji does not yet
+// exist it will be newly inserted into the database
+// followed by dereferencing the actual media file.
+//
+// Please note that even if an error is returned,
+// an emoji model may still be returned if the error
+// was only encountered during actual dereferencing.
+// In this case, it will act as a placeholder.
+func (d *Dereferencer) GetEmoji(
+ ctx context.Context,
+ shortcode string,
+ domain string,
+ remoteURL string,
+ info media.AdditionalEmojiInfo,
+ refresh bool,
+) (
+ *gtsmodel.Emoji,
+ error,
+) {
+ // Look for an existing emoji with shortcode domain.
+ emoji, err := d.state.DB.GetEmojiByShortcodeDomain(ctx,
+ shortcode,
+ domain,
+ )
+ if err != nil && !errors.Is(err, db.ErrNoEntries) {
+ return nil, gtserror.Newf("error fetching emoji from db: %w", err)
+ }
+
+ if emoji != nil {
+ // This was an existing emoji, pass to refresh func.
+ return d.RefreshEmoji(ctx, emoji, info, refresh)
+ }
+
+ if domain == "" {
+ // failed local lookup, will be db.ErrNoEntries.
+ return nil, gtserror.SetUnretrievable(err)
+ }
+
+ // Generate shortcode domain for locks + logging.
+ shortcodeDomain := shortcode + "@" + domain
+
+ // Ensure we have a valid remote URL.
+ url, err := url.Parse(remoteURL)
+ if err != nil {
+ err := gtserror.Newf("invalid image remote url %s for emoji %s: %w", remoteURL, shortcodeDomain, err)
+ return nil, err
+ }
+
+ // Acquire new instance account transport for emoji dereferencing.
+ tsport, err := d.transportController.NewTransportForUsername(ctx, "")
+ if err != nil {
+ err := gtserror.Newf("error getting instance transport: %w", err)
+ return nil, err
+ }
+
+ // Prepare data function to dereference remote emoji media.
+ data := func(context.Context) (io.ReadCloser, int64, error) {
+ return tsport.DereferenceMedia(ctx, url)
+ }
+
+ // Pass along for safe processing.
+ return d.processEmojiSafely(ctx,
+ shortcodeDomain,
+ func() (*media.ProcessingEmoji, error) {
+ return d.mediaManager.CreateEmoji(ctx,
+ shortcode,
+ domain,
+ data,
+ info,
+ )
+ },
+ )
+}
+
+// RefreshEmoji ensures that the given emoji is
+// up-to-date, both in terms of being cached in
+// in local instance storage, and compared to extra
+// information provided in media.AdditionEmojiInfo{}.
+// (note that is a no-op to pass in a local emoji).
+//
+// Please note that even if an error is returned,
+// an emoji model may still be returned if the error
+// was only encountered during actual dereferencing.
+// In this case, it will act as a placeholder.
+func (d *Dereferencer) RefreshEmoji(
+ ctx context.Context,
+ emoji *gtsmodel.Emoji,
+ info media.AdditionalEmojiInfo,
+ force bool,
+) (
+ *gtsmodel.Emoji,
+ error,
+) {
+ // Can't refresh local.
+ if emoji.IsLocal() {
+ return emoji, nil
+ }
+
+ // Check emoji is up-to-date
+ // with provided extra info.
+ switch {
+ case info.URI != nil &&
+ *info.URI != emoji.URI:
+ force = true
+ case info.ImageRemoteURL != nil &&
+ *info.ImageRemoteURL != emoji.ImageRemoteURL:
+ force = true
+ case info.ImageStaticRemoteURL != nil &&
+ *info.ImageStaticRemoteURL != emoji.ImageStaticRemoteURL:
+ force = true
+ }
+
+ // Check if needs updating.
+ if !force && *emoji.Cached {
+ return emoji, nil
+ }
+
+ // TODO: more finegrained freshness checks.
- // Ensure we have been passed a valid URL.
- derefURI, err := url.Parse(remoteURL)
+ // Generate shortcode domain for locks + logging.
+ shortcodeDomain := emoji.Shortcode + "@" + emoji.Domain
+
+ // Ensure we have a valid image remote URL.
+ url, err := url.Parse(emoji.ImageRemoteURL)
if err != nil {
- return nil, fmt.Errorf("GetRemoteEmoji: error parsing url for emoji %s: %s", shortcodeDomain, err)
+ err := gtserror.Newf("invalid image remote url %s for emoji %s: %w", emoji.ImageRemoteURL, shortcodeDomain, err)
+ return nil, err
}
- // Acquire derefs lock.
+ // Acquire new instance account transport for emoji dereferencing.
+ tsport, err := d.transportController.NewTransportForUsername(ctx, "")
+ if err != nil {
+ err := gtserror.Newf("error getting instance transport: %w", err)
+ return nil, err
+ }
+
+ // Prepare data function to dereference remote emoji media.
+ data := func(context.Context) (io.ReadCloser, int64, error) {
+ return tsport.DereferenceMedia(ctx, url)
+ }
+
+ // Pass along for safe processing.
+ return d.processEmojiSafely(ctx,
+ shortcodeDomain,
+ func() (*media.ProcessingEmoji, error) {
+ return d.mediaManager.RefreshEmoji(ctx,
+ emoji,
+ data,
+ info,
+ )
+ },
+ )
+}
+
+// processingEmojiSafely provides concurrency-safe processing of
+// an emoji with given shortcode+domain. if a copy of the emoji is
+// not already being processed, the given 'process' callback will
+// be used to generate new *media.ProcessingEmoji{} instance.
+func (d *Dereferencer) processEmojiSafely(
+ ctx context.Context,
+ shortcodeDomain string,
+ process func() (*media.ProcessingEmoji, error),
+) (
+ emoji *gtsmodel.Emoji,
+ err error,
+) {
+
+ // Acquire map lock.
d.derefEmojisMu.Lock()
// Ensure unlock only done once.
@@ -53,146 +214,118 @@ func (d *Dereferencer) GetRemoteEmoji(ctx context.Context, requestUser string, r
processing, ok := d.derefEmojis[shortcodeDomain]
if !ok {
- // Fetch a transport for current request user in order to perform request.
- tsport, err := d.transportController.NewTransportForUsername(ctx, requestUser)
+ // Start new processing emoji.
+ processing, err = process()
if err != nil {
- return nil, gtserror.Newf("couldn't create transport: %w", err)
+ return nil, err
}
-
- // Set the media data function to dereference emoji from URI.
- data := func(ctx context.Context) (io.ReadCloser, int64, error) {
- return tsport.DereferenceMedia(ctx, derefURI)
- }
-
- // Create new emoji processing request from the media manager.
- processing, err = d.mediaManager.PreProcessEmoji(ctx, data,
- shortcode,
- id,
- emojiURI,
- ai,
- refresh,
- )
- if err != nil {
- return nil, gtserror.Newf("error preprocessing emoji %s: %s", shortcodeDomain, err)
- }
-
- // Store media in map to mark as processing.
- d.derefEmojis[shortcodeDomain] = processing
-
- defer func() {
- // On exit safely remove emoji from map.
- d.derefEmojisMu.Lock()
- delete(d.derefEmojis, shortcodeDomain)
- d.derefEmojisMu.Unlock()
- }()
}
// Unlock map.
unlock()
- // Start emoji attachment loading (blocking call).
- if _, err := processing.LoadEmoji(ctx); err != nil {
- return nil, err
+ // Perform emoji load operation.
+ emoji, err = processing.Load(ctx)
+ if err != nil {
+ err = gtserror.Newf("error loading emoji %s: %w", shortcodeDomain, err)
+
+ // TODO: in time we should return checkable flags by gtserror.Is___()
+ // which can determine if loading error should allow remaining placeholder.
}
- return processing, nil
+ // Return a COPY of emoji.
+ emoji2 := new(gtsmodel.Emoji)
+ *emoji2 = *emoji
+ return emoji2, err
}
-func (d *Dereferencer) populateEmojis(ctx context.Context, rawEmojis []*gtsmodel.Emoji, requestingUsername string) ([]*gtsmodel.Emoji, error) {
- // At this point we should know:
- // * the AP uri of the emoji
- // * the domain of the emoji
- // * the shortcode of the emoji
- // * the remote URL of the image
- // This should be enough to dereference the emoji
- gotEmojis := make([]*gtsmodel.Emoji, 0, len(rawEmojis))
-
- for _, e := range rawEmojis {
- var gotEmoji *gtsmodel.Emoji
- var err error
- shortcodeDomain := e.Shortcode + "@" + e.Domain
-
- // check if we already know this emoji
- if e.ID != "" {
- // we had an ID for this emoji already, which means
- // it should be fleshed out already and we won't
- // have to get it from the database again
- gotEmoji = e
- } else if gotEmoji, err = d.state.DB.GetEmojiByShortcodeDomain(ctx, e.Shortcode, e.Domain); err != nil && err != db.ErrNoEntries {
- log.Errorf(ctx, "error checking database for emoji %s: %s", shortcodeDomain, err)
- continue
- }
-
- var refresh bool
+func (d *Dereferencer) fetchEmojis(
+ ctx context.Context,
+ existing []*gtsmodel.Emoji,
+ emojis []*gtsmodel.Emoji, // newly dereferenced
+) (
+ []*gtsmodel.Emoji,
+ bool, // any changes?
+ error,
+) {
+ // Track any changes.
+ changed := false
- if gotEmoji != nil {
- // we had the emoji already, but refresh it if necessary
- if e.UpdatedAt.Unix() > gotEmoji.ImageUpdatedAt.Unix() {
- log.Tracef(ctx, "emoji %s was updated since we last saw it, will refresh", shortcodeDomain)
- refresh = true
- }
+ for i, placeholder := range emojis {
+ // Look for an existing emoji with shortcode + domain.
+ existing, ok := getEmojiByShortcodeDomain(existing,
+ placeholder.Shortcode,
+ placeholder.Domain,
+ )
+ if ok && existing.ID != "" {
- if !refresh && (e.URI != gotEmoji.URI) {
- log.Tracef(ctx, "emoji %s changed URI since we last saw it, will refresh", shortcodeDomain)
- refresh = true
- }
+ // Check for any emoji changes that
+ // indicate we should force a refresh.
+ force := emojiChanged(existing, placeholder)
- if !refresh && (e.ImageRemoteURL != gotEmoji.ImageRemoteURL) {
- log.Tracef(ctx, "emoji %s changed image URL since we last saw it, will refresh", shortcodeDomain)
- refresh = true
- }
+ // Ensure that the existing emoji model is up-to-date and cached.
+ existing, err := d.RefreshEmoji(ctx, existing, media.AdditionalEmojiInfo{
- if !refresh {
- log.Tracef(ctx, "emoji %s is up to date, will not refresh", shortcodeDomain)
- } else {
- log.Tracef(ctx, "refreshing emoji %s", shortcodeDomain)
- emojiID := gotEmoji.ID // use existing ID
- processingEmoji, err := d.GetRemoteEmoji(ctx, requestingUsername, e.ImageRemoteURL, e.Shortcode, e.Domain, emojiID, e.URI, &media.AdditionalEmojiInfo{
- Domain: &e.Domain,
- ImageRemoteURL: &e.ImageRemoteURL,
- ImageStaticRemoteURL: &e.ImageStaticRemoteURL,
- Disabled: gotEmoji.Disabled,
- VisibleInPicker: gotEmoji.VisibleInPicker,
- }, refresh)
- if err != nil {
- log.Errorf(ctx, "couldn't refresh remote emoji %s: %s", shortcodeDomain, err)
- continue
- }
-
- if gotEmoji, err = processingEmoji.LoadEmoji(ctx); err != nil {
- log.Errorf(ctx, "couldn't load refreshed remote emoji %s: %s", shortcodeDomain, err)
- continue
- }
- }
- } else {
- // it's new! go get it!
- newEmojiID, err := id.NewRandomULID()
+ // Set latest values from placeholder.
+ URI: &placeholder.URI,
+ ImageRemoteURL: &placeholder.ImageRemoteURL,
+ ImageStaticRemoteURL: &placeholder.ImageStaticRemoteURL,
+ }, force)
if err != nil {
- log.Errorf(ctx, "error generating id for remote emoji %s: %s", shortcodeDomain, err)
- continue
- }
+ log.Errorf(ctx, "error refreshing emoji: %v", err)
- processingEmoji, err := d.GetRemoteEmoji(ctx, requestingUsername, e.ImageRemoteURL, e.Shortcode, e.Domain, newEmojiID, e.URI, &media.AdditionalEmojiInfo{
- Domain: &e.Domain,
- ImageRemoteURL: &e.ImageRemoteURL,
- ImageStaticRemoteURL: &e.ImageStaticRemoteURL,
- Disabled: e.Disabled,
- VisibleInPicker: e.VisibleInPicker,
- }, refresh)
- if err != nil {
- log.Errorf(ctx, "couldn't get remote emoji %s: %s", shortcodeDomain, err)
- continue
+ // specifically do NOT continue here,
+ // we already have a model, we don't
+ // want to drop it from the slice, just
+ // log that an update for it failed.
}
- if gotEmoji, err = processingEmoji.LoadEmoji(ctx); err != nil {
- log.Errorf(ctx, "couldn't load remote emoji %s: %s", shortcodeDomain, err)
+ // Set existing emoji.
+ emojis[i] = existing
+ continue
+ }
+
+ // Emojis changed!
+ changed = true
+
+ // Fetch this newly added emoji,
+ // this function handles the case
+ // of existing cached emojis and
+ // new ones requiring dereference.
+ emoji, err := d.GetEmoji(ctx,
+ placeholder.Shortcode,
+ placeholder.Domain,
+ placeholder.ImageRemoteURL,
+ media.AdditionalEmojiInfo{
+ URI: &placeholder.URI,
+ ImageRemoteURL: &placeholder.ImageRemoteURL,
+ ImageStaticRemoteURL: &placeholder.ImageStaticRemoteURL,
+ },
+ false,
+ )
+ if err != nil {
+ if emoji == nil {
+ log.Errorf(ctx, "error loading emoji %s: %v", placeholder.ImageRemoteURL, err)
continue
}
+
+ // non-fatal error occurred during loading, still use it.
+ log.Warnf(ctx, "partially loaded emoji: %v", err)
}
- // if we get here, we either had the emoji already or we successfully fetched it
- gotEmojis = append(gotEmojis, gotEmoji)
+ // Set updated emoji.
+ emojis[i] = emoji
+ }
+
+ for i := 0; i < len(emojis); {
+ if emojis[i].ID == "" {
+ // Remove failed emoji populations.
+ copy(emojis[i:], emojis[i+1:])
+ emojis = emojis[:len(emojis)-1]
+ continue
+ }
+ i++
}
- return gotEmojis, nil
+ return emojis, changed, nil
}
diff --git a/internal/federation/dereferencing/emoji_test.go b/internal/federation/dereferencing/emoji_test.go
index 08365741f..fdb815762 100644
--- a/internal/federation/dereferencing/emoji_test.go
+++ b/internal/federation/dereferencing/emoji_test.go
@@ -19,6 +19,7 @@ package dereferencing_test
import (
"context"
+ "fmt"
"testing"
"time"
@@ -32,48 +33,50 @@ type EmojiTestSuite struct {
func (suite *EmojiTestSuite) TestDereferenceEmojiBlocking() {
ctx := context.Background()
- fetchingAccount := suite.testAccounts["local_account_1"]
emojiImageRemoteURL := "http://example.org/media/emojis/1781772.gif"
emojiImageStaticRemoteURL := "http://example.org/media/emojis/1781772.gif"
emojiURI := "http://example.org/emojis/1781772"
emojiShortcode := "peglin"
- emojiID := "01GCBMGNZBKMEE1KTZ6PMJEW5D"
emojiDomain := "example.org"
emojiDisabled := false
emojiVisibleInPicker := false
- ai := &media.AdditionalEmojiInfo{
- Domain: &emojiDomain,
- ImageRemoteURL: &emojiImageRemoteURL,
- ImageStaticRemoteURL: &emojiImageStaticRemoteURL,
- Disabled: &emojiDisabled,
- VisibleInPicker: &emojiVisibleInPicker,
- }
-
- processingEmoji, err := suite.dereferencer.GetRemoteEmoji(ctx, fetchingAccount.Username, emojiImageRemoteURL, emojiShortcode, emojiDomain, emojiID, emojiURI, ai, false)
- suite.NoError(err)
-
- // make a blocking call to load the emoji from the in-process media
- emoji, err := processingEmoji.LoadEmoji(ctx)
+ emoji, err := suite.dereferencer.GetEmoji(
+ ctx,
+ emojiShortcode,
+ emojiDomain,
+ emojiImageRemoteURL,
+ media.AdditionalEmojiInfo{
+ URI: &emojiURI,
+ Domain: &emojiDomain,
+ ImageRemoteURL: &emojiImageRemoteURL,
+ ImageStaticRemoteURL: &emojiImageStaticRemoteURL,
+ Disabled: &emojiDisabled,
+ VisibleInPicker: &emojiVisibleInPicker,
+ },
+ false,
+ )
suite.NoError(err)
suite.NotNil(emoji)
- suite.Equal(emojiID, emoji.ID)
+ expectPath := fmt.Sprintf("/emoji/original/%s.gif", emoji.ID)
+ expectStaticPath := fmt.Sprintf("/emoji/static/%s.png", emoji.ID)
+
suite.WithinDuration(time.Now(), emoji.CreatedAt, 10*time.Second)
suite.WithinDuration(time.Now(), emoji.UpdatedAt, 10*time.Second)
suite.Equal(emojiShortcode, emoji.Shortcode)
suite.Equal(emojiDomain, emoji.Domain)
suite.Equal(emojiImageRemoteURL, emoji.ImageRemoteURL)
suite.Equal(emojiImageStaticRemoteURL, emoji.ImageStaticRemoteURL)
- suite.Contains(emoji.ImageURL, "/emoji/original/01GCBMGNZBKMEE1KTZ6PMJEW5D.gif")
- suite.Contains(emoji.ImageStaticURL, "emoji/static/01GCBMGNZBKMEE1KTZ6PMJEW5D.png")
- suite.Contains(emoji.ImagePath, "/emoji/original/01GCBMGNZBKMEE1KTZ6PMJEW5D.gif")
- suite.Contains(emoji.ImageStaticPath, "/emoji/static/01GCBMGNZBKMEE1KTZ6PMJEW5D.png")
+ suite.Contains(emoji.ImageURL, expectPath)
+ suite.Contains(emoji.ImageStaticURL, expectStaticPath)
+ suite.Contains(emoji.ImagePath, expectPath)
+ suite.Contains(emoji.ImageStaticPath, expectStaticPath)
suite.Equal("image/gif", emoji.ImageContentType)
suite.Equal("image/png", emoji.ImageStaticContentType)
suite.Equal(37796, emoji.ImageFileSize)
suite.Equal(7951, emoji.ImageStaticFileSize)
- suite.WithinDuration(time.Now(), emoji.ImageUpdatedAt, 10*time.Second)
+ suite.WithinDuration(time.Now(), emoji.UpdatedAt, 10*time.Second)
suite.False(*emoji.Disabled)
suite.Equal(emojiURI, emoji.URI)
suite.False(*emoji.VisibleInPicker)
diff --git a/internal/federation/dereferencing/media.go b/internal/federation/dereferencing/media.go
new file mode 100644
index 000000000..874107b13
--- /dev/null
+++ b/internal/federation/dereferencing/media.go
@@ -0,0 +1,215 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package dereferencing
+
+import (
+ "context"
+ "io"
+ "net/url"
+
+ "github.com/superseriousbusiness/gotosocial/internal/gtserror"
+ "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+ "github.com/superseriousbusiness/gotosocial/internal/media"
+)
+
+// GetMedia fetches the media at given remote URL by
+// dereferencing it. The passed accountID is used to
+// store it as being owned by that account. Additional
+// information to set on the media attachment may also
+// be provided.
+//
+// Please note that even if an error is returned,
+// a media model may still be returned if the error
+// was only encountered during actual dereferencing.
+// In this case, it will act as a placeholder.
+//
+// Also note that since account / status dereferencing is
+// already protected by per-uri locks, and that fediverse
+// media is generally not shared between accounts (etc),
+// there aren't any concurrency protections against multiple
+// insertion / dereferencing of media at remoteURL. Worst
+// case scenario, an extra media entry will be inserted
+// and the scheduled cleaner.Cleaner{} will catch it!
+func (d *Dereferencer) GetMedia(
+ ctx context.Context,
+ requestUser string,
+ accountID string, // media account owner
+ remoteURL string,
+ info media.AdditionalMediaInfo,
+) (
+ *gtsmodel.MediaAttachment,
+ error,
+) {
+ // Parse str as valid URL object.
+ url, err := url.Parse(remoteURL)
+ if err != nil {
+ return nil, gtserror.Newf("invalid remote media url %q: %v", remoteURL, err)
+ }
+
+ // Fetch transport for the provided request user from controller.
+ tsport, err := d.transportController.NewTransportForUsername(ctx,
+ requestUser,
+ )
+ if err != nil {
+ return nil, gtserror.Newf("failed getting transport for %s: %w", requestUser, err)
+ }
+
+ // Start processing remote attachment at URL.
+ processing, err := d.mediaManager.CreateMedia(
+ ctx,
+ accountID,
+ func(ctx context.Context) (io.ReadCloser, int64, error) {
+ return tsport.DereferenceMedia(ctx, url)
+ },
+ info,
+ )
+ if err != nil {
+ return nil, err
+ }
+
+ // Perform media load operation.
+ media, err := processing.Load(ctx)
+ if err != nil {
+ err = gtserror.Newf("error loading media %s: %w", media.RemoteURL, err)
+
+ // TODO: in time we should return checkable flags by gtserror.Is___()
+ // which can determine if loading error should allow remaining placeholder.
+ }
+
+ return media, err
+}
+
+// RefreshMedia ensures that given media is up-to-date,
+// both in terms of being cached in local instance,
+// storage and compared to extra info in information
+// in given gtsmodel.AdditionMediaInfo{}. This handles
+// the case of local emoji by returning early.
+//
+// Please note that even if an error is returned,
+// a media model may still be returned if the error
+// was only encountered during actual dereferencing.
+// In this case, it will act as a placeholder.
+//
+// Also note that since account / status dereferencing is
+// already protected by per-uri locks, and that fediverse
+// media is generally not shared between accounts (etc),
+// there aren't any concurrency protections against multiple
+// insertion / dereferencing of media at remoteURL. Worst
+// case scenario, an extra media entry will be inserted
+// and the scheduled cleaner.Cleaner{} will catch it!
+func (d *Dereferencer) RefreshMedia(
+ ctx context.Context,
+ requestUser string,
+ media *gtsmodel.MediaAttachment,
+ info media.AdditionalMediaInfo,
+ force bool,
+) (
+ *gtsmodel.MediaAttachment,
+ error,
+) {
+ // Can't refresh local.
+ if media.IsLocal() {
+ return media, nil
+ }
+
+ // Check emoji is up-to-date
+ // with provided extra info.
+ switch {
+ case info.Blurhash != nil &&
+ *info.Blurhash != media.Blurhash:
+ force = true
+ case info.Description != nil &&
+ *info.Description != media.Description:
+ force = true
+ case info.RemoteURL != nil &&
+ *info.RemoteURL != media.RemoteURL:
+ force = true
+ }
+
+ // Check if needs updating.
+ if !force && *media.Cached {
+ return media, nil
+ }
+
+ // TODO: more finegrained freshness checks.
+
+ // Ensure we have a valid remote URL.
+ url, err := url.Parse(media.RemoteURL)
+ if err != nil {
+ err := gtserror.Newf("invalid media remote url %s: %w", media.RemoteURL, err)
+ return nil, err
+ }
+
+ // Fetch transport for the provided request user from controller.
+ tsport, err := d.transportController.NewTransportForUsername(ctx,
+ requestUser,
+ )
+ if err != nil {
+ return nil, gtserror.Newf("failed getting transport for %s: %w", requestUser, err)
+ }
+
+ // Start processing remote attachment recache.
+ processing := d.mediaManager.RecacheMedia(
+ media,
+ func(ctx context.Context) (io.ReadCloser, int64, error) {
+ return tsport.DereferenceMedia(ctx, url)
+ },
+ )
+
+ // Perform media load operation.
+ media, err = processing.Load(ctx)
+ if err != nil {
+ err = gtserror.Newf("error loading media %s: %w", media.RemoteURL, err)
+
+ // TODO: in time we should return checkable flags by gtserror.Is___()
+ // which can determine if loading error should allow remaining placeholder.
+ }
+
+ return media, err
+}
+
+// updateAttachment handles the case of an existing media attachment
+// that *may* have changes or need recaching. it checks for changed
+// fields, updating in the database if so, and recaches uncached media.
+func (d *Dereferencer) updateAttachment(
+ ctx context.Context,
+ requestUser string,
+ existing *gtsmodel.MediaAttachment, // existing attachment
+ attach *gtsmodel.MediaAttachment, // (optional) changed media
+) (
+ *gtsmodel.MediaAttachment, // always set
+ error,
+) {
+ var info media.AdditionalMediaInfo
+
+ if attach != nil {
+ // Set optional extra information,
+ // (will later check for changes).
+ info.Description = &attach.Description
+ info.Blurhash = &attach.Blurhash
+ info.RemoteURL = &attach.RemoteURL
+ }
+
+ // Ensure media is cached.
+ return d.RefreshMedia(ctx,
+ requestUser,
+ existing,
+ info,
+ false,
+ )
+}
diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go
index add12c31f..406534457 100644
--- a/internal/federation/dereferencing/status.go
+++ b/internal/federation/dereferencing/status.go
@@ -33,7 +33,6 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/media"
- "github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
@@ -536,12 +535,12 @@ func (d *Dereferencer) enrichStatus(
}
// Ensure the status' media attachments are populated, passing in existing to check for changes.
- if err := d.fetchStatusAttachments(ctx, tsport, status, latestStatus); err != nil {
+ if err := d.fetchStatusAttachments(ctx, requestUser, status, latestStatus); err != nil {
return nil, nil, gtserror.Newf("error populating attachments for status %s: %w", uri, err)
}
- // Ensure the status' emoji attachments are populated, (changes are expected / okay).
- if err := d.fetchStatusEmojis(ctx, requestUser, latestStatus); err != nil {
+ // Ensure the status' emoji attachments are populated, passing in existing to check for changes.
+ if err := d.fetchStatusEmojis(ctx, status, latestStatus); err != nil {
return nil, nil, gtserror.Newf("error populating emojis for status %s: %w", uri, err)
}
@@ -643,79 +642,12 @@ func (d *Dereferencer) isPermittedStatus(
return onFail()
}
-// populateMentionTarget tries to populate the given
-// mention with the correct TargetAccount and (if not
-// yet set) TargetAccountURI, returning the populated
-// mention.
-//
-// Will check on the existing status if the mention
-// is already there and populated; if so, existing
-// mention will be returned along with `true`.
-//
-// Otherwise, this function will try to parse first
-// the Href of the mention, and then the namestring,
-// to see who it targets, and go fetch that account.
-func (d *Dereferencer) populateMentionTarget(
+func (d *Dereferencer) fetchStatusMentions(
ctx context.Context,
- mention *gtsmodel.Mention,
requestUser string,
- existing, status *gtsmodel.Status,
-) (
- *gtsmodel.Mention,
- bool, // True if mention already exists in the DB.
- error,
-) {
- // Mentions can be created using Name or Href.
- // Prefer Href (TargetAccountURI), fall back to Name.
- if mention.TargetAccountURI != "" {
- // Look for existing mention with this URI.
- // If we already have it we can return early.
- existingMention, ok := existing.GetMentionByTargetURI(mention.TargetAccountURI)
- if ok && existingMention.ID != "" {
- return existingMention, true, nil
- }
-
- // Ensure that mention account URI is parseable.
- accountURI, err := url.Parse(mention.TargetAccountURI)
- if err != nil {
- err = gtserror.Newf("invalid account uri %q: %w", mention.TargetAccountURI, err)
- return nil, false, err
- }
-
- // Ensure we have the account of the mention target dereferenced.
- mention.TargetAccount, _, err = d.getAccountByURI(ctx, requestUser, accountURI)
- if err != nil {
- err = gtserror.Newf("failed to dereference account %s: %w", accountURI, err)
- return nil, false, err
- }
- } else {
- // Href wasn't set. Find the target account using namestring.
- username, domain, err := util.ExtractNamestringParts(mention.NameString)
- if err != nil {
- err = gtserror.Newf("failed to parse namestring %s: %w", mention.NameString, err)
- return nil, false, err
- }
-
- mention.TargetAccount, _, err = d.getAccountByUsernameDomain(ctx, requestUser, username, domain)
- if err != nil {
- err = gtserror.Newf("failed to dereference account %s: %w", mention.NameString, err)
- return nil, false, err
- }
-
- // Look for existing mention with this URI.
- mention.TargetAccountURI = mention.TargetAccount.URI
- existingMention, ok := existing.GetMentionByTargetURI(mention.TargetAccountURI)
- if ok && existingMention.ID != "" {
- return existingMention, true, nil
- }
- }
-
- // At this point, mention.TargetAccountURI
- // and mention.TargetAccount must be set.
- return mention, false, nil
-}
-
-func (d *Dereferencer) fetchStatusMentions(ctx context.Context, requestUser string, existing, status *gtsmodel.Status) error {
+ existing *gtsmodel.Status,
+ status *gtsmodel.Status,
+) error {
// Allocate new slice to take the yet-to-be created mention IDs.
status.MentionIDs = make([]string, len(status.Mentions))
@@ -728,10 +660,10 @@ func (d *Dereferencer) fetchStatusMentions(ctx context.Context, requestUser stri
mention, alreadyExists, err = d.populateMentionTarget(
ctx,
- mention,
requestUser,
existing,
status,
+ mention,
)
if err != nil {
log.Errorf(ctx, "failed to derive mention: %v", err)
@@ -845,7 +777,11 @@ func (d *Dereferencer) threadStatus(ctx context.Context, status *gtsmodel.Status
return nil
}
-func (d *Dereferencer) fetchStatusTags(ctx context.Context, existing, status *gtsmodel.Status) error {
+func (d *Dereferencer) fetchStatusTags(
+ ctx context.Context,
+ existing *gtsmodel.Status,
+ status *gtsmodel.Status,
+) error {
// Allocate new slice to take the yet-to-be determined tag IDs.
status.TagIDs = make([]string, len(status.Tags))
@@ -900,7 +836,11 @@ func (d *Dereferencer) fetchStatusTags(ctx context.Context, existing, status *gt
return nil
}
-func (d *Dereferencer) fetchStatusPoll(ctx context.Context, existing, status *gtsmodel.Status) error {
+func (d *Dereferencer) fetchStatusPoll(
+ ctx context.Context,
+ existing *gtsmodel.Status,
+ status *gtsmodel.Status,
+) error {
var (
// insertStatusPoll generates ID and inserts the poll attached to status into the database.
insertStatusPoll = func(ctx context.Context, status *gtsmodel.Status) error {
@@ -990,19 +930,24 @@ func (d *Dereferencer) fetchStatusPoll(ctx context.Context, existing, status *gt
}
}
-func (d *Dereferencer) fetchStatusAttachments(ctx context.Context, tsport transport.Transport, existing, status *gtsmodel.Status) error {
+func (d *Dereferencer) fetchStatusAttachments(
+ ctx context.Context,
+ requestUser string,
+ existing *gtsmodel.Status,
+ status *gtsmodel.Status,
+) error {
// Allocate new slice to take the yet-to-be fetched attachment IDs.
status.AttachmentIDs = make([]string, len(status.Attachments))
for i := range status.Attachments {
- attachment := status.Attachments[i]
+ placeholder := status.Attachments[i]
// Look for existing media attachment with remote URL first.
- existing, ok := existing.GetAttachmentByRemoteURL(attachment.RemoteURL)
+ existing, ok := existing.GetAttachmentByRemoteURL(placeholder.RemoteURL)
if ok && existing.ID != "" {
// Ensure the existing media attachment is up-to-date and cached.
- existing, err := d.updateAttachment(ctx, tsport, existing, attachment)
+ existing, err := d.updateAttachment(ctx, requestUser, existing, placeholder)
if err != nil {
log.Errorf(ctx, "error updating existing attachment: %v", err)
@@ -1019,25 +964,25 @@ func (d *Dereferencer) fetchStatusAttachments(ctx context.Context, tsport transp
}
// Load this new media attachment.
- attachment, err := d.loadAttachment(
+ attachment, err := d.GetMedia(
ctx,
- tsport,
+ requestUser,
status.AccountID,
- attachment.RemoteURL,
- &media.AdditionalMediaInfo{
+ placeholder.RemoteURL,
+ media.AdditionalMediaInfo{
StatusID: &status.ID,
- RemoteURL: &attachment.RemoteURL,
- Description: &attachment.Description,
- Blurhash: &attachment.Blurhash,
+ RemoteURL: &placeholder.RemoteURL,
+ Description: &placeholder.Description,
+ Blurhash: &placeholder.Blurhash,
},
)
- if err != nil && attachment == nil {
- log.Errorf(ctx, "error loading attachment: %v", err)
- continue
- }
-
if err != nil {
- // A non-fatal error occurred during loading.
+ if attachment == nil {
+ log.Errorf(ctx, "error loading attachment %s: %v", placeholder.RemoteURL, err)
+ continue
+ }
+
+ // non-fatal error occurred during loading, still use it.
log.Warnf(ctx, "partially loaded attachment: %v", err)
}
@@ -1061,22 +1006,108 @@ func (d *Dereferencer) fetchStatusAttachments(ctx context.Context, tsport transp
return nil
}
-func (d *Dereferencer) fetchStatusEmojis(ctx context.Context, requestUser string, status *gtsmodel.Status) error {
- // Fetch the full-fleshed-out emoji objects for our status.
- emojis, err := d.populateEmojis(ctx, status.Emojis, requestUser)
+func (d *Dereferencer) fetchStatusEmojis(
+ ctx context.Context,
+ existing *gtsmodel.Status,
+ status *gtsmodel.Status,
+) error {
+ // Fetch the updated emojis for our status.
+ emojis, changed, err := d.fetchEmojis(ctx,
+ existing.Emojis,
+ status.Emojis,
+ )
if err != nil {
- return gtserror.Newf("failed to populate emojis: %w", err)
+ return gtserror.Newf("error fetching emojis: %w", err)
}
- // Iterate over and get their IDs.
- emojiIDs := make([]string, 0, len(emojis))
- for _, e := range emojis {
- emojiIDs = append(emojiIDs, e.ID)
+ if !changed {
+ // Use existing status emoji objects.
+ status.EmojiIDs = existing.EmojiIDs
+ status.Emojis = existing.Emojis
+ return nil
}
- // Set known emoji details.
+ // Set latest emojis.
status.Emojis = emojis
- status.EmojiIDs = emojiIDs
+
+ // Iterate over and set changed emoji IDs.
+ status.EmojiIDs = make([]string, len(emojis))
+ for i, emoji := range emojis {
+ status.EmojiIDs[i] = emoji.ID
+ }
return nil
}
+
+// populateMentionTarget tries to populate the given
+// mention with the correct TargetAccount and (if not
+// yet set) TargetAccountURI, returning the populated
+// mention.
+//
+// Will check on the existing status if the mention
+// is already there and populated; if so, existing
+// mention will be returned along with `true`.
+//
+// Otherwise, this function will try to parse first
+// the Href of the mention, and then the namestring,
+// to see who it targets, and go fetch that account.
+func (d *Dereferencer) populateMentionTarget(
+ ctx context.Context,
+ requestUser string,
+ existing *gtsmodel.Status,
+ status *gtsmodel.Status,
+ mention *gtsmodel.Mention,
+) (
+ *gtsmodel.Mention,
+ bool, // True if mention already exists in the DB.
+ error,
+) {
+ // Mentions can be created using Name or Href.
+ // Prefer Href (TargetAccountURI), fall back to Name.
+ if mention.TargetAccountURI != "" {
+ // Look for existing mention with this URI.
+ // If we already have it we can return early.
+ existingMention, ok := existing.GetMentionByTargetURI(mention.TargetAccountURI)
+ if ok && existingMention.ID != "" {
+ return existingMention, true, nil
+ }
+
+ // Ensure that mention account URI is parseable.
+ accountURI, err := url.Parse(mention.TargetAccountURI)
+ if err != nil {
+ err = gtserror.Newf("invalid account uri %q: %w", mention.TargetAccountURI, err)
+ return nil, false, err
+ }
+
+ // Ensure we have the account of the mention target dereferenced.
+ mention.TargetAccount, _, err = d.getAccountByURI(ctx, requestUser, accountURI)
+ if err != nil {
+ err = gtserror.Newf("failed to dereference account %s: %w", accountURI, err)
+ return nil, false, err
+ }
+ } else {
+ // Href wasn't set. Find the target account using namestring.
+ username, domain, err := util.ExtractNamestringParts(mention.NameString)
+ if err != nil {
+ err = gtserror.Newf("failed to parse namestring %s: %w", mention.NameString, err)
+ return nil, false, err
+ }
+
+ mention.TargetAccount, _, err = d.getAccountByUsernameDomain(ctx, requestUser, username, domain)
+ if err != nil {
+ err = gtserror.Newf("failed to dereference account %s: %w", mention.NameString, err)
+ return nil, false, err
+ }
+
+ // Look for existing mention with this URI.
+ mention.TargetAccountURI = mention.TargetAccount.URI
+ existingMention, ok := existing.GetMentionByTargetURI(mention.TargetAccountURI)
+ if ok && existingMention.ID != "" {
+ return existingMention, true, nil
+ }
+ }
+
+ // At this point, mention.TargetAccountURI
+ // and mention.TargetAccount must be set.
+ return mention, false, nil
+}
diff --git a/internal/federation/dereferencing/util.go b/internal/federation/dereferencing/util.go
index 5cb7a0106..297e90adc 100644
--- a/internal/federation/dereferencing/util.go
+++ b/internal/federation/dereferencing/util.go
@@ -18,120 +18,36 @@
package dereferencing
import (
- "context"
- "io"
- "net/url"
"slices"
- "github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
- "github.com/superseriousbusiness/gotosocial/internal/media"
- "github.com/superseriousbusiness/gotosocial/internal/transport"
- "github.com/superseriousbusiness/gotosocial/internal/util"
)
-// loadAttachment handles the case of a new media attachment
-// that requires loading. it stores and caches from given data.
-func (d *Dereferencer) loadAttachment(
- ctx context.Context,
- tsport transport.Transport,
- accountID string, // media account owner
- remoteURL string,
- info *media.AdditionalMediaInfo,
+// getEmojiByShortcodeDomain searches input slice
+// for emoji with given shortcode and domain.
+func getEmojiByShortcodeDomain(
+ emojis []*gtsmodel.Emoji,
+ shortcode string,
+ domain string,
) (
- *gtsmodel.MediaAttachment,
- error,
+ *gtsmodel.Emoji,
+ bool,
) {
- // Parse str as valid URL object.
- url, err := url.Parse(remoteURL)
- if err != nil {
- return nil, gtserror.Newf("invalid remote media url %q: %v", remoteURL, err)
- }
-
- // Start pre-processing remote media at remote URL.
- processing := d.mediaManager.PreProcessMedia(
- func(ctx context.Context) (io.ReadCloser, int64, error) {
- return tsport.DereferenceMedia(ctx, url)
- },
- accountID,
- info,
- )
-
- // Force attachment loading *right now*.
- return processing.LoadAttachment(ctx)
-}
-
-// updateAttachment handles the case of an existing media attachment
-// that *may* have changes or need recaching. it checks for changed
-// fields, updating in the database if so, and recaches uncached media.
-func (d *Dereferencer) updateAttachment(
- ctx context.Context,
- tsport transport.Transport,
- existing *gtsmodel.MediaAttachment, // existing attachment
- media *gtsmodel.MediaAttachment, // (optional) changed media
-) (
- *gtsmodel.MediaAttachment, // always set
- error,
-) {
- if media != nil {
- // Possible changed media columns.
- changed := make([]string, 0, 3)
-
- // Check if attachment description has changed.
- if existing.Description != media.Description {
- changed = append(changed, "description")
- existing.Description = media.Description
+ for _, emoji := range emojis {
+ if emoji.Shortcode == shortcode &&
+ emoji.Domain == domain {
+ return emoji, true
}
-
- // Check if attachment blurhash has changed (i.e. content change).
- if existing.Blurhash != media.Blurhash && media.Blurhash != "" {
- changed = append(changed, "blurhash", "cached")
- existing.Blurhash = media.Blurhash
- existing.Cached = util.Ptr(false)
- }
-
- if len(changed) > 0 {
- // Update the existing attachment model in the database.
- err := d.state.DB.UpdateAttachment(ctx, existing, changed...)
- if err != nil {
- return media, gtserror.Newf("error updating media: %w", err)
- }
- }
- }
-
- // Check if cached.
- if *existing.Cached {
- return existing, nil
- }
-
- // Parse str as valid URL object.
- url, err := url.Parse(existing.RemoteURL)
- if err != nil {
- return nil, gtserror.Newf("invalid remote media url %q: %v", media.RemoteURL, err)
}
+ return nil, false
+}
- // Start pre-processing remote media recaching from remote.
- processing, err := d.mediaManager.PreProcessMediaRecache(
- ctx,
- func(ctx context.Context) (io.ReadCloser, int64, error) {
- return tsport.DereferenceMedia(ctx, url)
- },
- existing.ID,
- )
- if err != nil {
- return nil, gtserror.Newf("error processing recache: %w", err)
- }
-
- // Force load attachment recache *right now*.
- recached, err := processing.LoadAttachment(ctx)
-
- // Always return the error we
- // receive, but ensure we return
- // most up-to-date media file.
- if recached != nil {
- return recached, err
- }
- return existing, err
+// emojiChanged returns whether an emoji has changed in a way
+// that indicates that it should be refetched and refreshed.
+func emojiChanged(existing, latest *gtsmodel.Emoji) bool {
+ return existing.URI != latest.URI ||
+ existing.ImageRemoteURL != latest.ImageRemoteURL ||
+ existing.ImageStaticRemoteURL != latest.ImageStaticRemoteURL
}
// pollChanged returns whether a poll has changed in way that
generated by cgit v1.2.3 (git 2.46.0) at 2025-07-25 11:27:55 +0000