[feature] Conversations API (#3013)

* Implement conversations API

* Sort and page conversations by last status ID

* Appease linter

* Fix deleting conversations and statuses

* Refactor to make migrations automatic

* Lint

* Update tests post-merge

* Fixes from live-fire testing

* Linter caught a format problem

* Refactor tests, fix cache

* Negative test for non-DMs

* Run conversations advanced migration on testrig startup as well as regular server startup

* Document (lack of) side effects of API method for deleting a conversation

* Make not-found check less nested for readability

* Rename PutConversation to UpsertConversation

* Use util.Ptr instead of IIFE

* Reduce cache used by conversations

* Remove unnecessary TableExpr/ColumnExpr

* Use struct tags for both unique constraints on Conversation

* Make it clear how paging with GetDirectStatusIDsBatch should be used

* Let conversation paging skip conversations it can't render

* Use Bun NewDropTable

* Convert delete raw query to Bun

* Convert update raw query to Bun

* Convert latestConversationStatusesTempTable raw query partially to Bun

* Convert conversationStatusesTempTable raw query partially to Bun

* Rename field used to store result of MaxDirectStatusID

* Move advanced migrations to their own tiny processor

* Catch up util function name with main

* Remove json.… wrappers

* Remove redundant check

* Combine error checks

* Replace map with slice of structs

* Address processor/type converter comments

- Add context info for errors
- Extract some common processor code into shared methods
- Move conversation eligibility check ahead of populating conversation

* Add error context when dropping temp tables

1 files changed, 49 insertions, 47 deletions

diff --git a/internal/config/config.go b/internal/config/config.go
index bffa5b455..1b8cf2759 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -191,53 +191,55 @@ type HTTPClientConfiguration struct {
 }
 
 type CacheConfiguration struct {
-	MemoryTarget                bytesize.Size `name:"memory-target"`
-	AccountMemRatio             float64       `name:"account-mem-ratio"`
-	AccountNoteMemRatio         float64       `name:"account-note-mem-ratio"`
-	AccountSettingsMemRatio     float64       `name:"account-settings-mem-ratio"`
-	AccountStatsMemRatio        float64       `name:"account-stats-mem-ratio"`
-	ApplicationMemRatio         float64       `name:"application-mem-ratio"`
-	BlockMemRatio               float64       `name:"block-mem-ratio"`
-	BlockIDsMemRatio            float64       `name:"block-ids-mem-ratio"`
-	BoostOfIDsMemRatio          float64       `name:"boost-of-ids-mem-ratio"`
-	ClientMemRatio              float64       `name:"client-mem-ratio"`
-	EmojiMemRatio               float64       `name:"emoji-mem-ratio"`
-	EmojiCategoryMemRatio       float64       `name:"emoji-category-mem-ratio"`
-	FilterMemRatio              float64       `name:"filter-mem-ratio"`
-	FilterKeywordMemRatio       float64       `name:"filter-keyword-mem-ratio"`
-	FilterStatusMemRatio        float64       `name:"filter-status-mem-ratio"`
-	FollowMemRatio              float64       `name:"follow-mem-ratio"`
-	FollowIDsMemRatio           float64       `name:"follow-ids-mem-ratio"`
-	FollowRequestMemRatio       float64       `name:"follow-request-mem-ratio"`
-	FollowRequestIDsMemRatio    float64       `name:"follow-request-ids-mem-ratio"`
-	InReplyToIDsMemRatio        float64       `name:"in-reply-to-ids-mem-ratio"`
-	InstanceMemRatio            float64       `name:"instance-mem-ratio"`
-	InteractionApprovalMemRatio float64       `name:"interaction-approval-mem-ratio"`
-	ListMemRatio                float64       `name:"list-mem-ratio"`
-	ListEntryMemRatio           float64       `name:"list-entry-mem-ratio"`
-	MarkerMemRatio              float64       `name:"marker-mem-ratio"`
-	MediaMemRatio               float64       `name:"media-mem-ratio"`
-	MentionMemRatio             float64       `name:"mention-mem-ratio"`
-	MoveMemRatio                float64       `name:"move-mem-ratio"`
-	NotificationMemRatio        float64       `name:"notification-mem-ratio"`
-	PollMemRatio                float64       `name:"poll-mem-ratio"`
-	PollVoteMemRatio            float64       `name:"poll-vote-mem-ratio"`
-	PollVoteIDsMemRatio         float64       `name:"poll-vote-ids-mem-ratio"`
-	ReportMemRatio              float64       `name:"report-mem-ratio"`
-	StatusMemRatio              float64       `name:"status-mem-ratio"`
-	StatusBookmarkMemRatio      float64       `name:"status-bookmark-mem-ratio"`
-	StatusBookmarkIDsMemRatio   float64       `name:"status-bookmark-ids-mem-ratio"`
-	StatusFaveMemRatio          float64       `name:"status-fave-mem-ratio"`
-	StatusFaveIDsMemRatio       float64       `name:"status-fave-ids-mem-ratio"`
-	TagMemRatio                 float64       `name:"tag-mem-ratio"`
-	ThreadMuteMemRatio          float64       `name:"thread-mute-mem-ratio"`
-	TokenMemRatio               float64       `name:"token-mem-ratio"`
-	TombstoneMemRatio           float64       `name:"tombstone-mem-ratio"`
-	UserMemRatio                float64       `name:"user-mem-ratio"`
-	UserMuteMemRatio            float64       `name:"user-mute-mem-ratio"`
-	UserMuteIDsMemRatio         float64       `name:"user-mute-ids-mem-ratio"`
-	WebfingerMemRatio           float64       `name:"webfinger-mem-ratio"`
-	VisibilityMemRatio          float64       `name:"visibility-mem-ratio"`
+	MemoryTarget                      bytesize.Size `name:"memory-target"`
+	AccountMemRatio                   float64       `name:"account-mem-ratio"`
+	AccountNoteMemRatio               float64       `name:"account-note-mem-ratio"`
+	AccountSettingsMemRatio           float64       `name:"account-settings-mem-ratio"`
+	AccountStatsMemRatio              float64       `name:"account-stats-mem-ratio"`
+	ApplicationMemRatio               float64       `name:"application-mem-ratio"`
+	BlockMemRatio                     float64       `name:"block-mem-ratio"`
+	BlockIDsMemRatio                  float64       `name:"block-ids-mem-ratio"`
+	BoostOfIDsMemRatio                float64       `name:"boost-of-ids-mem-ratio"`
+	ClientMemRatio                    float64       `name:"client-mem-ratio"`
+	ConversationMemRatio              float64       `name:"conversation-mem-ratio"`
+	ConversationLastStatusIDsMemRatio float64       `name:"conversation-last-status-ids-mem-ratio"`
+	EmojiMemRatio                     float64       `name:"emoji-mem-ratio"`
+	EmojiCategoryMemRatio             float64       `name:"emoji-category-mem-ratio"`
+	FilterMemRatio                    float64       `name:"filter-mem-ratio"`
+	FilterKeywordMemRatio             float64       `name:"filter-keyword-mem-ratio"`
+	FilterStatusMemRatio              float64       `name:"filter-status-mem-ratio"`
+	FollowMemRatio                    float64       `name:"follow-mem-ratio"`
+	FollowIDsMemRatio                 float64       `name:"follow-ids-mem-ratio"`
+	FollowRequestMemRatio             float64       `name:"follow-request-mem-ratio"`
+	FollowRequestIDsMemRatio          float64       `name:"follow-request-ids-mem-ratio"`
+	InReplyToIDsMemRatio              float64       `name:"in-reply-to-ids-mem-ratio"`
+	InstanceMemRatio                  float64       `name:"instance-mem-ratio"`
+	InteractionApprovalMemRatio       float64       `name:"interaction-approval-mem-ratio"`
+	ListMemRatio                      float64       `name:"list-mem-ratio"`
+	ListEntryMemRatio                 float64       `name:"list-entry-mem-ratio"`
+	MarkerMemRatio                    float64       `name:"marker-mem-ratio"`
+	MediaMemRatio                     float64       `name:"media-mem-ratio"`
+	MentionMemRatio                   float64       `name:"mention-mem-ratio"`
+	MoveMemRatio                      float64       `name:"move-mem-ratio"`
+	NotificationMemRatio              float64       `name:"notification-mem-ratio"`
+	PollMemRatio                      float64       `name:"poll-mem-ratio"`
+	PollVoteMemRatio                  float64       `name:"poll-vote-mem-ratio"`
+	PollVoteIDsMemRatio               float64       `name:"poll-vote-ids-mem-ratio"`
+	ReportMemRatio                    float64       `name:"report-mem-ratio"`
+	StatusMemRatio                    float64       `name:"status-mem-ratio"`
+	StatusBookmarkMemRatio            float64       `name:"status-bookmark-mem-ratio"`
+	StatusBookmarkIDsMemRatio         float64       `name:"status-bookmark-ids-mem-ratio"`
+	StatusFaveMemRatio                float64       `name:"status-fave-mem-ratio"`
+	StatusFaveIDsMemRatio             float64       `name:"status-fave-ids-mem-ratio"`
+	TagMemRatio                       float64       `name:"tag-mem-ratio"`
+	ThreadMuteMemRatio                float64       `name:"thread-mute-mem-ratio"`
+	TokenMemRatio                     float64       `name:"token-mem-ratio"`
+	TombstoneMemRatio                 float64       `name:"tombstone-mem-ratio"`
+	UserMemRatio                      float64       `name:"user-mem-ratio"`
+	UserMuteMemRatio                  float64       `name:"user-mute-mem-ratio"`
+	UserMuteIDsMemRatio               float64       `name:"user-mute-ids-mem-ratio"`
+	WebfingerMemRatio                 float64       `name:"webfinger-mem-ratio"`
+	VisibilityMemRatio                float64       `name:"visibility-mem-ratio"`
 }
 
 // MarshalMap will marshal current Configuration into a map structure (useful for JSON/TOML/YAML).