[feature] Conversations API (#3013)

* Implement conversations API

* Sort and page conversations by last status ID

* Appease linter

* Fix deleting conversations and statuses

* Refactor to make migrations automatic

* Lint

* Update tests post-merge

* Fixes from live-fire testing

* Linter caught a format problem

* Refactor tests, fix cache

* Negative test for non-DMs

* Run conversations advanced migration on testrig startup as well as regular server startup

* Document (lack of) side effects of API method for deleting a conversation

* Make not-found check less nested for readability

* Rename PutConversation to UpsertConversation

* Use util.Ptr instead of IIFE

* Reduce cache used by conversations

* Remove unnecessary TableExpr/ColumnExpr

* Use struct tags for both unique constraints on Conversation

* Make it clear how paging with GetDirectStatusIDsBatch should be used

* Let conversation paging skip conversations it can't render

* Use Bun NewDropTable

* Convert delete raw query to Bun

* Convert update raw query to Bun

* Convert latestConversationStatusesTempTable raw query partially to Bun

* Convert conversationStatusesTempTable raw query partially to Bun

* Rename field used to store result of MaxDirectStatusID

* Move advanced migrations to their own tiny processor

* Catch up util function name with main

* Remove json.… wrappers

* Remove redundant check

* Combine error checks

* Replace map with slice of structs

* Address processor/type converter comments

- Add context info for errors
- Extract some common processor code into shared methods
- Move conversation eligibility check ahead of populating conversation

* Add error context when dropping temp tables

1 files changed, 131 insertions, 0 deletions

diff --git a/internal/processing/conversations/migrate.go b/internal/processing/conversations/migrate.go
new file mode 100644
index 000000000..959ffcca4
--- /dev/null
+++ b/internal/processing/conversations/migrate.go
@@ -0,0 +1,131 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package conversations
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/id"
+	"github.com/superseriousbusiness/gotosocial/internal/log"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+const advancedMigrationID = "20240611190733_add_conversations"
+const statusBatchSize = 100
+
+type AdvancedMigrationState struct {
+	MinID          string
+	MaxIDInclusive string
+}
+
+func (p *Processor) MigrateDMsToConversations(ctx context.Context) error {
+	advancedMigration, err := p.state.DB.GetAdvancedMigration(ctx, advancedMigrationID)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		return gtserror.Newf("couldn't get advanced migration with ID %s: %w", advancedMigrationID, err)
+	}
+	state := AdvancedMigrationState{}
+	if advancedMigration != nil {
+		// There was a previous migration.
+		if *advancedMigration.Finished {
+			// This migration has already been run to completion; we don't need to run it again.
+			return nil
+		}
+		// Otherwise, pick up where we left off.
+		if err := json.Unmarshal(advancedMigration.StateJSON, &state); err != nil {
+			// This should never happen.
+			return gtserror.Newf("couldn't deserialize advanced migration state from JSON: %w", err)
+		}
+	} else {
+		// Start at the beginning.
+		state.MinID = id.Lowest
+
+		// Find the max ID of all existing statuses.
+		// This will be the last one we migrate;
+		// newer ones will be handled by the normal conversation flow.
+		state.MaxIDInclusive, err = p.state.DB.MaxDirectStatusID(ctx)
+		if err != nil {
+			return gtserror.Newf("couldn't get max DM status ID for migration: %w", err)
+		}
+
+		// Save a new advanced migration record.
+		advancedMigration = &gtsmodel.AdvancedMigration{
+			ID:       advancedMigrationID,
+			Finished: util.Ptr(false),
+		}
+		if advancedMigration.StateJSON, err = json.Marshal(state); err != nil {
+			// This should never happen.
+			return gtserror.Newf("couldn't serialize advanced migration state to JSON: %w", err)
+		}
+		if err := p.state.DB.PutAdvancedMigration(ctx, advancedMigration); err != nil {
+			return gtserror.Newf("couldn't save state for advanced migration with ID %s: %w", advancedMigrationID, err)
+		}
+	}
+
+	log.Info(ctx, "migrating DMs to conversations…")
+
+	// In batches, get all statuses up to and including the max ID,
+	// and update conversations for each in order.
+	for {
+		// Get status IDs for this batch.
+		statusIDs, err := p.state.DB.GetDirectStatusIDsBatch(ctx, state.MinID, state.MaxIDInclusive, statusBatchSize)
+		if err != nil {
+			return gtserror.Newf("couldn't get DM status ID batch for migration: %w", err)
+		}
+		if len(statusIDs) == 0 {
+			break
+		}
+		log.Infof(ctx, "migrating %d DMs starting after %s", len(statusIDs), state.MinID)
+
+		// Load the batch by IDs.
+		statuses, err := p.state.DB.GetStatusesByIDs(ctx, statusIDs)
+		if err != nil {
+			return gtserror.Newf("couldn't get DM statuses for migration: %w", err)
+		}
+
+		// Update conversations for each status. Don't generate notifications.
+		for _, status := range statuses {
+			if _, err := p.UpdateConversationsForStatus(ctx, status); err != nil {
+				return gtserror.Newf("couldn't update conversations for status %s during migration: %w", status.ID, err)
+			}
+		}
+
+		// Save the migration state with the new min ID.
+		state.MinID = statusIDs[len(statusIDs)-1]
+		if advancedMigration.StateJSON, err = json.Marshal(state); err != nil {
+			// This should never happen.
+			return gtserror.Newf("couldn't serialize advanced migration state to JSON: %w", err)
+		}
+		if err := p.state.DB.PutAdvancedMigration(ctx, advancedMigration); err != nil {
+			return gtserror.Newf("couldn't save state for advanced migration with ID %s: %w", advancedMigrationID, err)
+		}
+	}
+
+	// Mark the migration as finished.
+	advancedMigration.Finished = util.Ptr(true)
+	if err := p.state.DB.PutAdvancedMigration(ctx, advancedMigration); err != nil {
+		return gtserror.Newf("couldn't save state for advanced migration with ID %s: %w", advancedMigrationID, err)
+	}
+
+	log.Info(ctx, "finished migrating DMs to conversations.")
+	return nil
+}