[feature] Add List functionality (#1802)

* start working on lists

* further list work

* test list db functions nicely

* more work on lists

* peepoopeepoo

* poke

* start list timeline func

* we're getting there lads

* couldn't be me working on stuff... could it?

* hook up handlers

* fiddling

* weeee

* woah

* screaming, pissing

* fix streaming being a whiny baby

* lint, small test fix, swagger

* tidying up, testing

* fucked! by the linter

* move timelines to state like a boss

* add timeline start to tests using state

* invalidate lists

7 files changed, 595 insertions, 0 deletions

diff --git a/internal/processing/list/create.go b/internal/processing/list/create.go
new file mode 100644
index 000000000..10dec1050
--- /dev/null
+++ b/internal/processing/list/create.go
@@ -0,0 +1,50 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package list
+
+import (
+	"context"
+	"errors"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/id"
+)
+
+// Create creates one a new list for the given account, using the provided parameters.
+// These params should have already been validated by the time they reach this function.
+func (p *Processor) Create(ctx context.Context, account *gtsmodel.Account, title string, repliesPolicy gtsmodel.RepliesPolicy) (*apimodel.List, gtserror.WithCode) {
+	list := &gtsmodel.List{
+		ID:            id.NewULID(),
+		Title:         title,
+		AccountID:     account.ID,
+		RepliesPolicy: repliesPolicy,
+	}
+
+	if err := p.state.DB.PutList(ctx, list); err != nil {
+		if errors.Is(err, db.ErrAlreadyExists) {
+			err = errors.New("you already have a list with this title")
+			return nil, gtserror.NewErrorConflict(err, err.Error())
+		}
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	return p.apiList(ctx, list)
+}
diff --git a/internal/processing/list/delete.go b/internal/processing/list/delete.go
new file mode 100644
index 000000000..1c8ee5700
--- /dev/null
+++ b/internal/processing/list/delete.go
@@ -0,0 +1,46 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package list
+
+import (
+	"context"
+
+	"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+// Delete deletes one list for the given account.
+func (p *Processor) Delete(ctx context.Context, account *gtsmodel.Account, id string) gtserror.WithCode {
+	list, errWithCode := p.getList(
+		// Use barebones ctx; no embedded
+		// structs necessary for this call.
+		gtscontext.SetBarebones(ctx),
+		account.ID,
+		id,
+	)
+	if errWithCode != nil {
+		return errWithCode
+	}
+
+	if err := p.state.DB.DeleteListByID(ctx, list.ID); err != nil {
+		return gtserror.NewErrorInternalError(err)
+	}
+
+	return nil
+}
diff --git a/internal/processing/list/get.go b/internal/processing/list/get.go
new file mode 100644
index 000000000..3f124fe7c
--- /dev/null
+++ b/internal/processing/list/get.go
@@ -0,0 +1,155 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package list
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/log"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+// Get returns the api model of one list with the given ID.
+func (p *Processor) Get(ctx context.Context, account *gtsmodel.Account, id string) (*apimodel.List, gtserror.WithCode) {
+	list, errWithCode := p.getList(
+		// Use barebones ctx; no embedded
+		// structs necessary for this call.
+		gtscontext.SetBarebones(ctx),
+		account.ID,
+		id,
+	)
+	if errWithCode != nil {
+		return nil, errWithCode
+	}
+
+	return p.apiList(ctx, list)
+}
+
+// GetMultiple returns multiple lists created by the given account, sorted by list ID DESC (newest first).
+func (p *Processor) GetAll(ctx context.Context, account *gtsmodel.Account) ([]*apimodel.List, gtserror.WithCode) {
+	lists, err := p.state.DB.GetListsForAccountID(
+		// Use barebones ctx; no embedded
+		// structs necessary for simple GET.
+		gtscontext.SetBarebones(ctx),
+		account.ID,
+	)
+	if err != nil {
+		if errors.Is(err, db.ErrNoEntries) {
+			return nil, nil
+		}
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	apiLists := make([]*apimodel.List, 0, len(lists))
+	for _, list := range lists {
+		apiList, errWithCode := p.apiList(ctx, list)
+		if errWithCode != nil {
+			return nil, errWithCode
+		}
+
+		apiLists = append(apiLists, apiList)
+	}
+
+	return apiLists, nil
+}
+
+// GetListAccounts returns accounts that are in the given list, owned by the given account.
+// The additional parameters can be used for paging.
+func (p *Processor) GetListAccounts(
+	ctx context.Context,
+	account *gtsmodel.Account,
+	listID string,
+	maxID string,
+	sinceID string,
+	minID string,
+	limit int,
+) (*apimodel.PageableResponse, gtserror.WithCode) {
+	// Ensure list exists + is owned by requesting account.
+	if _, errWithCode := p.getList(ctx, account.ID, listID); errWithCode != nil {
+		return nil, errWithCode
+	}
+
+	// To know which accounts are in the list,
+	// we need to first get requested list entries.
+	listEntries, err := p.state.DB.GetListEntries(ctx, listID, maxID, sinceID, minID, limit)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		err = fmt.Errorf("GetListAccounts: error getting list entries: %w", err)
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	count := len(listEntries)
+	if count == 0 {
+		// No list entries means no accounts.
+		return util.EmptyPageableResponse(), nil
+	}
+
+	var (
+		items          = make([]interface{}, count)
+		nextMaxIDValue string
+		prevMinIDValue string
+	)
+
+	// For each list entry, we want the account it points to.
+	// To get this, we need to first get the follow that the
+	// list entry pertains to, then extract the target account
+	// from that follow.
+	//
+	// We do paging not by account ID, but by list entry ID.
+	for i, listEntry := range listEntries {
+		if i == count-1 {
+			nextMaxIDValue = listEntry.ID
+		}
+
+		if i == 0 {
+			prevMinIDValue = listEntry.ID
+		}
+
+		if err := p.state.DB.PopulateListEntry(ctx, listEntry); err != nil {
+			log.Debugf(ctx, "skipping list entry because of error populating it: %q", err)
+			continue
+		}
+
+		if err := p.state.DB.PopulateFollow(ctx, listEntry.Follow); err != nil {
+			log.Debugf(ctx, "skipping list entry because of error populating follow: %q", err)
+			continue
+		}
+
+		apiAccount, err := p.tc.AccountToAPIAccountPublic(ctx, listEntry.Follow.TargetAccount)
+		if err != nil {
+			log.Debugf(ctx, "skipping list entry because of error converting follow target account: %q", err)
+			continue
+		}
+
+		items[i] = apiAccount
+	}
+
+	return util.PackagePageableResponse(util.PageableResponseParams{
+		Items:          items,
+		Path:           "api/v1/lists/" + listID + "/accounts",
+		NextMaxIDValue: nextMaxIDValue,
+		PrevMinIDValue: prevMinIDValue,
+		Limit:          limit,
+	})
+}
diff --git a/internal/processing/list/list.go b/internal/processing/list/list.go
new file mode 100644
index 000000000..f192beb60
--- /dev/null
+++ b/internal/processing/list/list.go
@@ -0,0 +1,35 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package list
+
+import (
+	"github.com/superseriousbusiness/gotosocial/internal/state"
+	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+)
+
+type Processor struct {
+	state *state.State
+	tc    typeutils.TypeConverter
+}
+
+func New(state *state.State, tc typeutils.TypeConverter) Processor {
+	return Processor{
+		state: state,
+		tc:    tc,
+	}
+}
diff --git a/internal/processing/list/update.go b/internal/processing/list/update.go
new file mode 100644
index 000000000..656af1f78
--- /dev/null
+++ b/internal/processing/list/update.go
@@ -0,0 +1,73 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package list
+
+import (
+	"context"
+	"errors"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+// Update updates one list for the given account, using the provided parameters.
+// These params should have already been validated by the time they reach this function.
+func (p *Processor) Update(
+	ctx context.Context,
+	account *gtsmodel.Account,
+	id string,
+	title *string,
+	repliesPolicy *gtsmodel.RepliesPolicy,
+) (*apimodel.List, gtserror.WithCode) {
+	list, errWithCode := p.getList(
+		// Use barebones ctx; no embedded
+		// structs necessary for this call.
+		gtscontext.SetBarebones(ctx),
+		account.ID,
+		id,
+	)
+	if errWithCode != nil {
+		return nil, errWithCode
+	}
+
+	// Only update columns we're told to update.
+	columns := make([]string, 0, 2)
+
+	if title != nil {
+		list.Title = *title
+		columns = append(columns, "title")
+	}
+
+	if repliesPolicy != nil {
+		list.RepliesPolicy = *repliesPolicy
+		columns = append(columns, "replies_policy")
+	}
+
+	if err := p.state.DB.UpdateList(ctx, list, columns...); err != nil {
+		if errors.Is(err, db.ErrAlreadyExists) {
+			err = errors.New("you already have a list with this title")
+			return nil, gtserror.NewErrorConflict(err, err.Error())
+		}
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	return p.apiList(ctx, list)
+}
diff --git a/internal/processing/list/updateentries.go b/internal/processing/list/updateentries.go
new file mode 100644
index 000000000..6dcb951a7
--- /dev/null
+++ b/internal/processing/list/updateentries.go
@@ -0,0 +1,151 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package list
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/id"
+)
+
+// AddToList adds targetAccountIDs to the given list, if valid.
+func (p *Processor) AddToList(ctx context.Context, account *gtsmodel.Account, listID string, targetAccountIDs []string) gtserror.WithCode {
+	// Ensure this list exists + account owns it.
+	list, errWithCode := p.getList(ctx, account.ID, listID)
+	if errWithCode != nil {
+		return errWithCode
+	}
+
+	// Pre-assemble list of entries to add. We *could* add these
+	// one by one as we iterate through accountIDs, but according
+	// to the Mastodon API we should only add them all once we know
+	// they're all valid, no partial updates.
+	listEntries := make([]*gtsmodel.ListEntry, 0, len(targetAccountIDs))
+
+	// Check each targetAccountID is valid.
+	//   - Follow must exist.
+	//   - Follow must not already be in the given list.
+	for _, targetAccountID := range targetAccountIDs {
+		// Ensure follow exists.
+		follow, err := p.state.DB.GetFollow(ctx, account.ID, targetAccountID)
+		if err != nil {
+			if errors.Is(err, db.ErrNoEntries) {
+				err = fmt.Errorf("you do not follow account %s", targetAccountID)
+				return gtserror.NewErrorNotFound(err, err.Error())
+			}
+			return gtserror.NewErrorInternalError(err)
+		}
+
+		// Ensure followID not already in list.
+		// This particular call to isInList will
+		// never error, so just check entryID.
+		entryID, _ := isInList(
+			list,
+			follow.ID,
+			func(listEntry *gtsmodel.ListEntry) (string, error) {
+				// Looking for the listEntry follow ID.
+				return listEntry.FollowID, nil
+			},
+		)
+
+		// Empty entryID means entry with given
+		// followID wasn't found in the list.
+		if entryID != "" {
+			err = fmt.Errorf("account with id %s is already in list %s with entryID %s", targetAccountID, listID, entryID)
+			return gtserror.NewErrorUnprocessableEntity(err, err.Error())
+		}
+
+		// Entry wasn't in the list, we can add it.
+		listEntries = append(listEntries, &gtsmodel.ListEntry{
+			ID:       id.NewULID(),
+			ListID:   listID,
+			FollowID: follow.ID,
+		})
+	}
+
+	// If we get to here we can assume all
+	// entries are valid, so try to add them.
+	if err := p.state.DB.PutListEntries(ctx, listEntries); err != nil {
+		if errors.Is(err, db.ErrAlreadyExists) {
+			err = fmt.Errorf("one or more errors inserting list entries: %w", err)
+			return gtserror.NewErrorUnprocessableEntity(err, err.Error())
+		}
+		return gtserror.NewErrorInternalError(err)
+	}
+
+	return nil
+}
+
+// RemoveFromList removes targetAccountIDs from the given list, if valid.
+func (p *Processor) RemoveFromList(ctx context.Context, account *gtsmodel.Account, listID string, targetAccountIDs []string) gtserror.WithCode {
+	// Ensure this list exists + account owns it.
+	list, errWithCode := p.getList(ctx, account.ID, listID)
+	if errWithCode != nil {
+		return errWithCode
+	}
+
+	// For each targetAccountID, we want to check if
+	// a follow with that targetAccountID is in the
+	// given list. If it is in there, we want to remove
+	// it from the list.
+	for _, targetAccountID := range targetAccountIDs {
+		// Check if targetAccountID is
+		// on a follow in the list.
+		entryID, err := isInList(
+			list,
+			targetAccountID,
+			func(listEntry *gtsmodel.ListEntry) (string, error) {
+				// We need the follow so populate this
+				// entry, if it's not already populated.
+				if err := p.state.DB.PopulateListEntry(ctx, listEntry); err != nil {
+					return "", err
+				}
+
+				// Looking for the list entry targetAccountID.
+				return listEntry.Follow.TargetAccountID, nil
+			},
+		)
+
+		// Error may be returned here if there was an issue
+		// populating the list entry. We only return on proper
+		// DB errors, we can just skip no entry errors.
+		if err != nil && !errors.Is(err, db.ErrNoEntries) {
+			err = fmt.Errorf("error checking if targetAccountID %s was in list %s: %w", targetAccountID, listID, err)
+			return gtserror.NewErrorInternalError(err)
+		}
+
+		if entryID == "" {
+			// There was an errNoEntries or targetAccount
+			// wasn't in this list anyway, so we can skip it.
+			continue
+		}
+
+		// TargetAccount was in the list, remove the entry.
+		if err := p.state.DB.DeleteListEntry(ctx, entryID); err != nil && !errors.Is(err, db.ErrNoEntries) {
+			err = fmt.Errorf("error removing list entry %s from list %s: %w", entryID, listID, err)
+			return gtserror.NewErrorInternalError(err)
+		}
+	}
+
+	return nil
+}
diff --git a/internal/processing/list/util.go b/internal/processing/list/util.go
new file mode 100644
index 000000000..6186f58c7
--- /dev/null
+++ b/internal/processing/list/util.go
@@ -0,0 +1,85 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package list
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+// getList is a shortcut to get one list from the database and
+// check that it's owned by the given accountID. Will return
+// appropriate errors so caller doesn't need to bother.
+func (p *Processor) getList(ctx context.Context, accountID string, listID string) (*gtsmodel.List, gtserror.WithCode) {
+	list, err := p.state.DB.GetListByID(ctx, listID)
+	if err != nil {
+		if errors.Is(err, db.ErrNoEntries) {
+			// List doesn't seem to exist.
+			return nil, gtserror.NewErrorNotFound(err)
+		}
+		// Real database error.
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	if list.AccountID != accountID {
+		err = fmt.Errorf("list with id %s does not belong to account %s", list.ID, accountID)
+		return nil, gtserror.NewErrorNotFound(err)
+	}
+
+	return list, nil
+}
+
+// apiList is a shortcut to return the API version of the given
+// list, or return an appropriate error if conversion fails.
+func (p *Processor) apiList(ctx context.Context, list *gtsmodel.List) (*apimodel.List, gtserror.WithCode) {
+	apiList, err := p.tc.ListToAPIList(ctx, list)
+	if err != nil {
+		return nil, gtserror.NewErrorInternalError(fmt.Errorf("error converting list to api: %w", err))
+	}
+
+	return apiList, nil
+}
+
+// isInList check if thisID is equal to the result of thatID
+// for any entry in the given list.
+//
+// Will return the id of the listEntry if true, empty if false,
+// or an error if the result of thatID returns an error.
+func isInList(
+	list *gtsmodel.List,
+	thisID string,
+	getThatID func(listEntry *gtsmodel.ListEntry) (string, error),
+) (string, error) {
+	for _, listEntry := range list.ListEntries {
+		thatID, err := getThatID(listEntry)
+		if err != nil {
+			return "", err
+		}
+
+		if thisID == thatID {
+			return listEntry.ID, nil
+		}
+	}
+	return "", nil
+}