[performance] overhaul struct (+ result) caching library for simplicity, performance and multiple-result lookups (#2535)

* rewrite cache library as codeberg.org/gruf/go-structr, implement in gotosocial

* use actual go-structr release version (not just commit hash)

* revert go toolchain changes (damn you go for auto changing this)

* fix go mod woes

* ensure %w is used in calls to errs.Appendf()

* fix error checking

* fix possible panic

* remove unnecessary start/stop functions, move to main Cache{} struct, add note regarding which caches require start/stop

* fix copy-paste artifact... :innocent:

* fix all comment copy-paste artifacts

* remove dropID() function, now we can just use slices.DeleteFunc()

* use util.Deduplicate() instead of collate(), move collate to util

* move orderByIDs() to util package and "generify"

* add a util.DeleteIf() function, use this to delete entries on failed population

* use slices.DeleteFunc() instead of util.DeleteIf() (i had the logic mixed up in my head somehow lol)

* add note about how collate differs from deduplicate

3 files changed, 28 insertions, 47 deletions

diff --git a/internal/typeutils/astointernal.go b/internal/typeutils/astointernal.go
index 5bcb35d75..ec17527c4 100644
--- a/internal/typeutils/astointernal.go
+++ b/internal/typeutils/astointernal.go
@@ -794,7 +794,6 @@ func (c *Converter) getASAttributedToAccount(ctx context.Context, id string, wit
 	}
 
 	return account, nil
-
 }
 
 func (c *Converter) getASObjectAccount(ctx context.Context, id string, with ap.WithObject) (*gtsmodel.Account, error) {
diff --git a/internal/typeutils/internaltoas.go b/internal/typeutils/internaltoas.go
index c88fd2e11..dc25babaa 100644
--- a/internal/typeutils/internaltoas.go
+++ b/internal/typeutils/internaltoas.go
@@ -491,7 +491,7 @@ func (c *Converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (ap.Stat
 
 	// tag -- mentions
 	mentions := s.Mentions
-	if len(s.MentionIDs) > len(mentions) {
+	if len(s.MentionIDs) != len(mentions) {
 		mentions, err = c.state.DB.GetMentions(ctx, s.MentionIDs)
 		if err != nil {
 			return nil, gtserror.Newf("error getting mentions: %w", err)
@@ -507,14 +507,10 @@ func (c *Converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (ap.Stat
 
 	// tag -- emojis
 	emojis := s.Emojis
-	if len(s.EmojiIDs) > len(emojis) {
-		emojis = []*gtsmodel.Emoji{}
-		for _, emojiID := range s.EmojiIDs {
-			emoji, err := c.state.DB.GetEmojiByID(ctx, emojiID)
-			if err != nil {
-				return nil, gtserror.Newf("error getting emoji %s from database: %w", emojiID, err)
-			}
-			emojis = append(emojis, emoji)
+	if len(s.EmojiIDs) != len(emojis) {
+		emojis, err = c.state.DB.GetEmojisByIDs(ctx, s.EmojiIDs)
+		if err != nil {
+			return nil, gtserror.Newf("error getting emojis from database: %w", err)
 		}
 	}
 	for _, emoji := range emojis {
@@ -527,7 +523,7 @@ func (c *Converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (ap.Stat
 
 	// tag -- hashtags
 	hashtags := s.Tags
-	if len(s.TagIDs) > len(hashtags) {
+	if len(s.TagIDs) != len(hashtags) {
 		hashtags, err = c.state.DB.GetTags(ctx, s.TagIDs)
 		if err != nil {
 			return nil, gtserror.Newf("error getting tags: %w", err)
@@ -623,14 +619,10 @@ func (c *Converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (ap.Stat
 	// attachments
 	attachmentProp := streams.NewActivityStreamsAttachmentProperty()
 	attachments := s.Attachments
-	if len(s.AttachmentIDs) > len(attachments) {
-		attachments = []*gtsmodel.MediaAttachment{}
-		for _, attachmentID := range s.AttachmentIDs {
-			attachment, err := c.state.DB.GetAttachmentByID(ctx, attachmentID)
-			if err != nil {
-				return nil, gtserror.Newf("error getting attachment %s from database: %w", attachmentID, err)
-			}
-			attachments = append(attachments, attachment)
+	if len(s.AttachmentIDs) != len(attachments) {
+		attachments, err = c.state.DB.GetAttachmentsByIDs(ctx, s.AttachmentIDs)
+		if err != nil {
+			return nil, gtserror.Newf("error getting attachments from database: %w", err)
 		}
 	}
 	for _, a := range attachments {
diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go
index 75247f411..941b9e866 100644
--- a/internal/typeutils/internaltofrontend.go
+++ b/internal/typeutils/internaltofrontend.go
@@ -1563,20 +1563,15 @@ func (c *Converter) PollToAPIPoll(ctx context.Context, requester *gtsmodel.Accou
 func (c *Converter) convertAttachmentsToAPIAttachments(ctx context.Context, attachments []*gtsmodel.MediaAttachment, attachmentIDs []string) ([]*apimodel.Attachment, error) {
 	var errs gtserror.MultiError
 
-	if len(attachments) == 0 {
+	if len(attachments) == 0 && len(attachmentIDs) > 0 {
 		// GTS model attachments were not populated
 
-		// Preallocate expected GTS slice
-		attachments = make([]*gtsmodel.MediaAttachment, 0, len(attachmentIDs))
+		var err error
 
 		// Fetch GTS models for attachment IDs
-		for _, id := range attachmentIDs {
-			attachment, err := c.state.DB.GetAttachmentByID(ctx, id)
-			if err != nil {
-				errs.Appendf("error fetching attachment %s from database: %v", id, err)
-				continue
-			}
-			attachments = append(attachments, attachment)
+		attachments, err = c.state.DB.GetAttachmentsByIDs(ctx, attachmentIDs)
+		if err != nil {
+			errs.Appendf("error fetching attachments from database: %w", err)
 		}
 	}
 
@@ -1587,7 +1582,7 @@ func (c *Converter) convertAttachmentsToAPIAttachments(ctx context.Context, atta
 	for _, attachment := range attachments {
 		apiAttachment, err := c.AttachmentToAPIAttachment(ctx, attachment)
 		if err != nil {
-			errs.Appendf("error converting attchment %s to api attachment: %v", attachment.ID, err)
+			errs.Appendf("error converting attchment %s to api attachment: %w", attachment.ID, err)
 			continue
 		}
 		apiAttachments = append(apiAttachments, &apiAttachment)
@@ -1600,20 +1595,15 @@ func (c *Converter) convertAttachmentsToAPIAttachments(ctx context.Context, atta
 func (c *Converter) convertEmojisToAPIEmojis(ctx context.Context, emojis []*gtsmodel.Emoji, emojiIDs []string) ([]apimodel.Emoji, error) {
 	var errs gtserror.MultiError
 
-	if len(emojis) == 0 {
+	if len(emojis) == 0 && len(emojiIDs) > 0 {
 		// GTS model attachments were not populated
 
-		// Preallocate expected GTS slice
-		emojis = make([]*gtsmodel.Emoji, 0, len(emojiIDs))
+		var err error
 
 		// Fetch GTS models for emoji IDs
-		for _, id := range emojiIDs {
-			emoji, err := c.state.DB.GetEmojiByID(ctx, id)
-			if err != nil {
-				errs.Appendf("error fetching emoji %s from database: %v", id, err)
-				continue
-			}
-			emojis = append(emojis, emoji)
+		emojis, err = c.state.DB.GetEmojisByIDs(ctx, emojiIDs)
+		if err != nil {
+			errs.Appendf("error fetching emojis from database: %w", err)
 		}
 	}
 
@@ -1624,7 +1614,7 @@ func (c *Converter) convertEmojisToAPIEmojis(ctx context.Context, emojis []*gtsm
 	for _, emoji := range emojis {
 		apiEmoji, err := c.EmojiToAPIEmoji(ctx, emoji)
 		if err != nil {
-			errs.Appendf("error converting emoji %s to api emoji: %v", emoji.ID, err)
+			errs.Appendf("error converting emoji %s to api emoji: %w", emoji.ID, err)
 			continue
 		}
 		apiEmojis = append(apiEmojis, apiEmoji)
@@ -1637,7 +1627,7 @@ func (c *Converter) convertEmojisToAPIEmojis(ctx context.Context, emojis []*gtsm
 func (c *Converter) convertMentionsToAPIMentions(ctx context.Context, mentions []*gtsmodel.Mention, mentionIDs []string) ([]apimodel.Mention, error) {
 	var errs gtserror.MultiError
 
-	if len(mentions) == 0 {
+	if len(mentions) == 0 && len(mentionIDs) > 0 {
 		var err error
 
 		// GTS model mentions were not populated
@@ -1645,7 +1635,7 @@ func (c *Converter) convertMentionsToAPIMentions(ctx context.Context, mentions [
 		// Fetch GTS models for mention IDs
 		mentions, err = c.state.DB.GetMentions(ctx, mentionIDs)
 		if err != nil {
-			errs.Appendf("error fetching mentions from database: %v", err)
+			errs.Appendf("error fetching mentions from database: %w", err)
 		}
 	}
 
@@ -1656,7 +1646,7 @@ func (c *Converter) convertMentionsToAPIMentions(ctx context.Context, mentions [
 	for _, mention := range mentions {
 		apiMention, err := c.MentionToAPIMention(ctx, mention)
 		if err != nil {
-			errs.Appendf("error converting mention %s to api mention: %v", mention.ID, err)
+			errs.Appendf("error converting mention %s to api mention: %w", mention.ID, err)
 			continue
 		}
 		apiMentions = append(apiMentions, apiMention)
@@ -1669,12 +1659,12 @@ func (c *Converter) convertMentionsToAPIMentions(ctx context.Context, mentions [
 func (c *Converter) convertTagsToAPITags(ctx context.Context, tags []*gtsmodel.Tag, tagIDs []string) ([]apimodel.Tag, error) {
 	var errs gtserror.MultiError
 
-	if len(tags) == 0 {
+	if len(tags) == 0 && len(tagIDs) > 0 {
 		var err error
 
 		tags, err = c.state.DB.GetTags(ctx, tagIDs)
 		if err != nil {
-			errs.Appendf("error fetching tags from database: %v", err)
+			errs.Appendf("error fetching tags from database: %w", err)
 		}
 	}
 
@@ -1685,7 +1675,7 @@ func (c *Converter) convertTagsToAPITags(ctx context.Context, tags []*gtsmodel.T
 	for _, tag := range tags {
 		apiTag, err := c.TagToAPITag(ctx, tag, false)
 		if err != nil {
-			errs.Appendf("error converting tag %s to api tag: %v", tag.ID, err)
+			errs.Appendf("error converting tag %s to api tag: %w", tag.ID, err)
 			continue
 		}
 		apiTags = append(apiTags, apiTag)