[feature] Implement exclusive lists (#3280)

Fixes #2616

2 files changed, 434 insertions, 46 deletions

diff --git a/internal/processing/workers/fromclientapi_test.go b/internal/processing/workers/fromclientapi_test.go
index d330e4c2b..cc8801e1c 100644
--- a/internal/processing/workers/fromclientapi_test.go
+++ b/internal/processing/workers/fromclientapi_test.go
@@ -1527,6 +1527,317 @@ func (suite *FromClientAPITestSuite) TestProcessCreateBoostWithFollowedHashtagAn
 	)
 }
 
+// A public status with a hashtag followed by a local user who follows the author and has them on an exclusive list
+// should end up in the following user's timeline for that list, but not their home timeline.
+func (suite *FromClientAPITestSuite) TestProcessCreateStatusWithAuthorOnExclusiveList() {
+	testStructs := testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	var (
+		ctx              = context.Background()
+		postingAccount   = suite.testAccounts["local_account_2"]
+		receivingAccount = suite.testAccounts["local_account_1"]
+		testList         = suite.testLists["local_account_1_list_1"]
+		streams          = suite.openStreams(ctx,
+			testStructs.Processor,
+			receivingAccount,
+			[]string{testList.ID},
+		)
+		homeStream = streams[stream.TimelineHome]
+		listStream = streams[stream.TimelineList+":"+testList.ID]
+
+		// postingAccount posts a new public status not mentioning anyone.
+		status = suite.newStatus(
+			ctx,
+			testStructs.State,
+			postingAccount,
+			gtsmodel.VisibilityPublic,
+			nil,
+			nil,
+			nil,
+			false,
+			nil,
+		)
+	)
+
+	// Setup: make the list exclusive.
+	// We modify the existing list rather than create a new one, so that there's only one list in play for this test.
+	list := new(gtsmodel.List)
+	*list = *testList
+	list.Exclusive = util.Ptr(true)
+	if err := testStructs.State.DB.UpdateList(ctx, list); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process the new status.
+	if err := testStructs.Processor.Workers().ProcessFromClientAPI(
+		ctx,
+		&messages.FromClientAPI{
+			APObjectType:   ap.ObjectNote,
+			APActivityType: ap.ActivityCreate,
+			GTSModel:       status,
+			Origin:         postingAccount,
+		},
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Check status in list stream.
+	suite.checkStreamed(
+		listStream,
+		true,
+		"",
+		stream.EventTypeUpdate,
+	)
+
+	// Check status not in home stream.
+	suite.checkStreamed(
+		homeStream,
+		false,
+		"",
+		"",
+	)
+}
+
+// A public status with a hashtag followed by a local user who follows the author and has them on an exclusive list
+// should end up in the following user's timeline for that list, but not their home timeline.
+// This should happen regardless of whether the author is on any of the following user's *non*-exclusive lists.
+func (suite *FromClientAPITestSuite) TestProcessCreateStatusWithAuthorOnExclusiveAndNonExclusiveLists() {
+	testStructs := testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	var (
+		ctx               = context.Background()
+		postingAccount    = suite.testAccounts["local_account_2"]
+		receivingAccount  = suite.testAccounts["local_account_1"]
+		testInclusiveList = suite.testLists["local_account_1_list_1"]
+		testExclusiveList = &gtsmodel.List{
+			ID:            id.NewULID(),
+			Title:         "Cool Ass Posters From This Instance (exclusive)",
+			AccountID:     receivingAccount.ID,
+			RepliesPolicy: gtsmodel.RepliesPolicyFollowed,
+			Exclusive:     util.Ptr(true),
+		}
+		testFollow               = suite.testFollows["local_account_1_local_account_2"]
+		testExclusiveListEntries = []*gtsmodel.ListEntry{
+			{
+				ID:       id.NewULID(),
+				ListID:   testExclusiveList.ID,
+				FollowID: testFollow.ID,
+			},
+		}
+		streams = suite.openStreams(ctx,
+			testStructs.Processor,
+			receivingAccount,
+			[]string{
+				testInclusiveList.ID,
+				testExclusiveList.ID,
+			},
+		)
+		homeStream          = streams[stream.TimelineHome]
+		inclusiveListStream = streams[stream.TimelineList+":"+testInclusiveList.ID]
+		exclusiveListStream = streams[stream.TimelineList+":"+testExclusiveList.ID]
+
+		// postingAccount posts a new public status not mentioning anyone.
+		status = suite.newStatus(
+			ctx,
+			testStructs.State,
+			postingAccount,
+			gtsmodel.VisibilityPublic,
+			nil,
+			nil,
+			nil,
+			false,
+			nil,
+		)
+	)
+
+	// Precondition: the pre-existing inclusive list should actually be inclusive.
+	// This should be the case if we reset the DB correctly between tests in this file.
+	{
+		list, err := testStructs.State.DB.GetListByID(ctx, testInclusiveList.ID)
+		if err != nil {
+			suite.FailNow(err.Error())
+		}
+		if *list.Exclusive {
+			suite.FailNowf(
+				"test precondition failed: list %s should be inclusive, but isn't",
+				testInclusiveList.ID,
+			)
+		}
+	}
+
+	// Setup: create the exclusive list and its list entry.
+	if err := testStructs.State.DB.PutList(ctx, testExclusiveList); err != nil {
+		suite.FailNow(err.Error())
+	}
+	if err := testStructs.State.DB.PutListEntries(ctx, testExclusiveListEntries); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process the new status.
+	if err := testStructs.Processor.Workers().ProcessFromClientAPI(
+		ctx,
+		&messages.FromClientAPI{
+			APObjectType:   ap.ObjectNote,
+			APActivityType: ap.ActivityCreate,
+			GTSModel:       status,
+			Origin:         postingAccount,
+		},
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Check status in inclusive list stream.
+	suite.checkStreamed(
+		inclusiveListStream,
+		true,
+		"",
+		stream.EventTypeUpdate,
+	)
+
+	// Check status in exclusive list stream.
+	suite.checkStreamed(
+		exclusiveListStream,
+		true,
+		"",
+		stream.EventTypeUpdate,
+	)
+
+	// Check status not in home stream.
+	suite.checkStreamed(
+		homeStream,
+		false,
+		"",
+		"",
+	)
+}
+
+// A public status with a hashtag followed by a local user who follows the author and has them on an exclusive list
+// should end up in the following user's timeline for that list, but not their home timeline.
+// When they have notifications on for that user, they should be notified.
+func (suite *FromClientAPITestSuite) TestProcessCreateStatusWithAuthorOnExclusiveListAndNotificationsOn() {
+	testStructs := testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	var (
+		ctx              = context.Background()
+		postingAccount   = suite.testAccounts["local_account_2"]
+		receivingAccount = suite.testAccounts["local_account_1"]
+		testFollow       = suite.testFollows["local_account_1_local_account_2"]
+		testList         = suite.testLists["local_account_1_list_1"]
+		streams          = suite.openStreams(ctx,
+			testStructs.Processor,
+			receivingAccount,
+			[]string{testList.ID},
+		)
+		homeStream  = streams[stream.TimelineHome]
+		listStream  = streams[stream.TimelineList+":"+testList.ID]
+		notifStream = streams[stream.TimelineNotifications]
+
+		// postingAccount posts a new public status not mentioning anyone.
+		status = suite.newStatus(
+			ctx,
+			testStructs.State,
+			postingAccount,
+			gtsmodel.VisibilityPublic,
+			nil,
+			nil,
+			nil,
+			false,
+			nil,
+		)
+	)
+
+	// Setup: Update the follow from receiving account -> posting account so
+	// that receiving account wants notifs when posting account posts.
+	follow := new(gtsmodel.Follow)
+	*follow = *testFollow
+	follow.Notify = util.Ptr(true)
+	if err := testStructs.State.DB.UpdateFollow(ctx, follow); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Setup: make the list exclusive.
+	list := new(gtsmodel.List)
+	*list = *testList
+	list.Exclusive = util.Ptr(true)
+	if err := testStructs.State.DB.UpdateList(ctx, list); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process the new status.
+	if err := testStructs.Processor.Workers().ProcessFromClientAPI(
+		ctx,
+		&messages.FromClientAPI{
+			APObjectType:   ap.ObjectNote,
+			APActivityType: ap.ActivityCreate,
+			GTSModel:       status,
+			Origin:         postingAccount,
+		},
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Check status in list stream.
+	suite.checkStreamed(
+		listStream,
+		true,
+		"",
+		stream.EventTypeUpdate,
+	)
+
+	// Wait for a notification to appear for the status.
+	var notif *gtsmodel.Notification
+	if !testrig.WaitFor(func() bool {
+		var err error
+		notif, err = testStructs.State.DB.GetNotification(
+			ctx,
+			gtsmodel.NotificationStatus,
+			receivingAccount.ID,
+			postingAccount.ID,
+			status.ID,
+		)
+		return err == nil
+	}) {
+		suite.FailNow("timed out waiting for new status notification")
+	}
+
+	apiNotif, err := testStructs.TypeConverter.NotificationToAPINotification(ctx, notif, nil, nil)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	notifJSON, err := json.Marshal(apiNotif)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Check message in notification stream.
+	suite.checkStreamed(
+		notifStream,
+		true,
+		string(notifJSON),
+		stream.EventTypeNotification,
+	)
+
+	// Check *notification* for status in home stream.
+	suite.checkStreamed(
+		homeStream,
+		true,
+		string(notifJSON),
+		stream.EventTypeNotification,
+	)
+
+	// Status itself should not be in home stream.
+	suite.checkStreamed(
+		homeStream,
+		false,
+		"",
+		"",
+	)
+}
+
 // Updating a public status with a hashtag followed by a local user who does not otherwise follow the author
 // should stream a status update to the tag-following user's home timeline.
 func (suite *FromClientAPITestSuite) TestProcessUpdateStatusWithFollowedHashtag() {
diff --git a/internal/processing/workers/surfacetimeline.go b/internal/processing/workers/surfacetimeline.go
index c0987effd..81544d928 100644
--- a/internal/processing/workers/surfacetimeline.go
+++ b/internal/processing/workers/surfacetimeline.go
@@ -34,7 +34,7 @@ import (
 )
 
 // timelineAndNotifyStatus inserts the given status into the HOME
-// and LIST timelines of accounts that follow the status author,
+// and/or LIST timelines of accounts that follow the status author,
 // as well as the HOME timelines of accounts that follow tags used by the status.
 //
 // It will also handle notifications for any mentions attached to
@@ -100,6 +100,7 @@ func (s *Surface) timelineAndNotifyStatus(ctx context.Context, status *gtsmodel.
 // new status, if the status is eligible for notification.
 //
 // Returns a list of accounts which had this status inserted into their home timelines.
+// This will be used to prevent duplicate inserts when handling followed tags.
 func (s *Surface) timelineAndNotifyStatusForFollowers(
 	ctx context.Context,
 	status *gtsmodel.Status,
@@ -118,8 +119,13 @@ func (s *Surface) timelineAndNotifyStatusForFollowers(
 		// it's a reblog, whether follower account wants to see reblogs.
 		//
 		// If it's not timelineable, we can just stop early, since lists
-		// are prettymuch subsets of the home timeline, so if it shouldn't
+		// are pretty much subsets of the home timeline, so if it shouldn't
 		// appear there, it shouldn't appear in lists either.
+		//
+		// Exclusive lists don't change this:
+		// if something is hometimelineable according to this filter,
+		// it's also eligible to appear in exclusive lists,
+		// even if it ultimately doesn't appear on the home timeline.
 		timelineable, err := s.VisFilter.StatusHomeTimelineable(
 			ctx, follow.Account, status,
 		)
@@ -141,7 +147,7 @@ func (s *Surface) timelineAndNotifyStatusForFollowers(
 
 		// Add status to any relevant lists
 		// for this follow, if applicable.
-		s.listTimelineStatusForFollow(
+		exclusive, listTimelined := s.listTimelineStatusForFollow(
 			ctx,
 			status,
 			follow,
@@ -152,27 +158,32 @@ func (s *Surface) timelineAndNotifyStatusForFollowers(
 
 		// Add status to home timeline for owner
 		// of this follow, if applicable.
-		homeTimelined, err := s.timelineStatus(
-			ctx,
-			s.State.Timelines.Home.IngestOne,
-			follow.AccountID, // home timelines are keyed by account ID
-			follow.Account,
-			status,
-			stream.TimelineHome,
-			filters,
-			mutes,
-		)
-		if err != nil {
-			errs.Appendf("error home timelining status: %w", err)
-			continue
+		homeTimelined := false
+		if !exclusive {
+			homeTimelined, err = s.timelineStatus(
+				ctx,
+				s.State.Timelines.Home.IngestOne,
+				follow.AccountID, // home timelines are keyed by account ID
+				follow.Account,
+				status,
+				stream.TimelineHome,
+				filters,
+				mutes,
+			)
+			if err != nil {
+				errs.Appendf("error home timelining status: %w", err)
+				continue
+			}
+			if homeTimelined {
+				homeTimelinedAccountIDs = append(homeTimelinedAccountIDs, follow.AccountID)
+			}
 		}
 
-		if !homeTimelined {
-			// If status wasn't added to home
-			// timeline, we shouldn't notify it.
+		if !(homeTimelined || listTimelined) {
+			// If status wasn't added to home or list
+			// timelines, we shouldn't notify it.
 			continue
 		}
-		homeTimelinedAccountIDs = append(homeTimelinedAccountIDs, follow.AccountID)
 
 		if !*follow.Notify {
 			// This follower doesn't have notifs
@@ -188,7 +199,7 @@ func (s *Surface) timelineAndNotifyStatusForFollowers(
 		// If we reach here, we know:
 		//
 		//   - This status is hometimelineable.
-		//   - This status was added to the home timeline for this follower.
+		//   - This status was added to the home timeline and/or list timelines for this follower.
 		//   - This follower wants to be notified when this account posts.
 		//   - This is a top-level post (not a reply or boost).
 		//
@@ -208,6 +219,10 @@ func (s *Surface) timelineAndNotifyStatusForFollowers(
 
 // listTimelineStatusForFollow puts the given status
 // in any eligible lists owned by the given follower.
+//
+// It returns whether the status was added to any lists,
+// and whether the status author is on any exclusive lists
+// (in which case the status shouldn't be added to the home timeline).
 func (s *Surface) listTimelineStatusForFollow(
 	ctx context.Context,
 	status *gtsmodel.Status,
@@ -215,7 +230,7 @@ func (s *Surface) listTimelineStatusForFollow(
 	errs *gtserror.MultiError,
 	filters []*gtsmodel.Filter,
 	mutes *usermute.CompiledUserMuteList,
-) {
+) (bool, bool) {
 	// To put this status in appropriate list timelines,
 	// we need to get each listEntry that pertains to
 	// this follow. Then, we want to iterate through all
@@ -223,18 +238,19 @@ func (s *Surface) listTimelineStatusForFollow(
 	// that the entry belongs to if it meets criteria for
 	// inclusion in the list.
 
-	// Get every list entry that targets this follow's ID.
-	listEntries, err := s.State.DB.GetListEntriesForFollowID(
-		// We only need the list IDs.
-		gtscontext.SetBarebones(ctx),
-		follow.ID,
-	)
-	if err != nil && !errors.Is(err, db.ErrNoEntries) {
-		errs.Appendf("error getting list entries: %w", err)
-		return
+	listEntries, err := s.getListEntries(ctx, follow)
+	if err != nil {
+		errs.Append(err)
+		return false, false
+	}
+	exclusive, err := s.isAnyListExclusive(ctx, listEntries)
+	if err != nil {
+		errs.Append(err)
+		return false, false
 	}
 
 	// Check eligibility for each list entry (if any).
+	listTimelined := false
 	for _, listEntry := range listEntries {
 		eligible, err := s.listEligible(ctx, listEntry, status)
 		if err != nil {
@@ -250,7 +266,7 @@ func (s *Surface) listTimelineStatusForFollow(
 		// At this point we are certain this status
 		// should be included in the timeline of the
 		// list that this list entry belongs to.
-		if _, err := s.timelineStatus(
+		timelined, err := s.timelineStatus(
 			ctx,
 			s.State.Timelines.List.IngestOne,
 			listEntry.ListID, // list timelines are keyed by list ID
@@ -259,11 +275,59 @@ func (s *Surface) listTimelineStatusForFollow(
 			stream.TimelineList+":"+listEntry.ListID, // key streamType to this specific list
 			filters,
 			mutes,
-		); err != nil {
+		)
+		if err != nil {
 			errs.Appendf("error adding status to timeline for list %s: %w", listEntry.ListID, err)
 			// implicit continue
 		}
+		listTimelined = listTimelined || timelined
+	}
+
+	return exclusive, listTimelined
+}
+
+// getListEntries returns list entries for a given follow.
+func (s *Surface) getListEntries(ctx context.Context, follow *gtsmodel.Follow) ([]*gtsmodel.ListEntry, error) {
+	// Get every list entry that targets this follow's ID.
+	listEntries, err := s.State.DB.GetListEntriesForFollowID(
+		// We only need the list IDs.
+		gtscontext.SetBarebones(ctx),
+		follow.ID,
+	)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		return nil, gtserror.Newf("DB error getting list entries: %v", err)
+	}
+	return listEntries, nil
+}
+
+// isAnyListExclusive determines whether any provided list entry corresponds to an exclusive list.
+func (s *Surface) isAnyListExclusive(ctx context.Context, listEntries []*gtsmodel.ListEntry) (bool, error) {
+	if len(listEntries) == 0 {
+		return false, nil
+	}
+
+	listIDs := make([]string, 0, len(listEntries))
+	for _, listEntry := range listEntries {
+		listIDs = append(listIDs, listEntry.ListID)
+	}
+	lists, err := s.State.DB.GetListsByIDs(
+		// We only need the list exclusive flags.
+		gtscontext.SetBarebones(ctx),
+		listIDs,
+	)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		return false, gtserror.Newf("DB error getting lists for list entries: %v", err)
+	}
+
+	if len(lists) == 0 {
+		return false, nil
+	}
+	for _, list := range lists {
+		if *list.Exclusive {
+			return true, nil
+		}
 	}
+	return false, nil
 }
 
 // getFiltersAndMutes returns an account's filters and mutes.
@@ -643,8 +707,13 @@ func (s *Surface) timelineStatusUpdateForFollowers(
 		// it's a reblog, whether follower account wants to see reblogs.
 		//
 		// If it's not timelineable, we can just stop early, since lists
-		// are prettymuch subsets of the home timeline, so if it shouldn't
+		// are pretty much subsets of the home timeline, so if it shouldn't
 		// appear there, it shouldn't appear in lists either.
+		//
+		// Exclusive lists don't change this:
+		// if something is hometimelineable according to this filter,
+		// it's also eligible to appear in exclusive lists,
+		// even if it ultimately doesn't appear on the home timeline.
 		timelineable, err := s.VisFilter.StatusHomeTimelineable(
 			ctx, follow.Account, status,
 		)
@@ -666,7 +735,7 @@ func (s *Surface) timelineStatusUpdateForFollowers(
 
 		// Add status to any relevant lists
 		// for this follow, if applicable.
-		s.listTimelineStatusUpdateForFollow(
+		exclusive := s.listTimelineStatusUpdateForFollow(
 			ctx,
 			status,
 			follow,
@@ -675,6 +744,10 @@ func (s *Surface) timelineStatusUpdateForFollowers(
 			mutes,
 		)
 
+		if exclusive {
+			continue
+		}
+
 		// Add status to home timeline for owner
 		// of this follow, if applicable.
 		homeTimelined, err := s.timelineStreamStatusUpdate(
@@ -689,7 +762,6 @@ func (s *Surface) timelineStatusUpdateForFollowers(
 			errs.Appendf("error home timelining status: %w", err)
 			continue
 		}
-
 		if homeTimelined {
 			homeTimelinedAccountIDs = append(homeTimelinedAccountIDs, follow.AccountID)
 		}
@@ -700,6 +772,9 @@ func (s *Surface) timelineStatusUpdateForFollowers(
 
 // listTimelineStatusUpdateForFollow pushes edits of the given status
 // into any eligible lists streams opened by the given follower.
+//
+// It returns whether the status author is on any exclusive lists
+// (in which case the status shouldn't be added to the home timeline).
 func (s *Surface) listTimelineStatusUpdateForFollow(
 	ctx context.Context,
 	status *gtsmodel.Status,
@@ -707,7 +782,7 @@ func (s *Surface) listTimelineStatusUpdateForFollow(
 	errs *gtserror.MultiError,
 	filters []*gtsmodel.Filter,
 	mutes *usermute.CompiledUserMuteList,
-) {
+) bool {
 	// To put this status in appropriate list timelines,
 	// we need to get each listEntry that pertains to
 	// this follow. Then, we want to iterate through all
@@ -715,15 +790,15 @@ func (s *Surface) listTimelineStatusUpdateForFollow(
 	// that the entry belongs to if it meets criteria for
 	// inclusion in the list.
 
-	// Get every list entry that targets this follow's ID.
-	listEntries, err := s.State.DB.GetListEntriesForFollowID(
-		// We only need the list IDs.
-		gtscontext.SetBarebones(ctx),
-		follow.ID,
-	)
-	if err != nil && !errors.Is(err, db.ErrNoEntries) {
-		errs.Appendf("error getting list entries: %w", err)
-		return
+	listEntries, err := s.getListEntries(ctx, follow)
+	if err != nil {
+		errs.Append(err)
+		return false
+	}
+	exclusive, err := s.isAnyListExclusive(ctx, listEntries)
+	if err != nil {
+		errs.Append(err)
+		return false
 	}
 
 	// Check eligibility for each list entry (if any).
@@ -754,6 +829,8 @@ func (s *Surface) listTimelineStatusUpdateForFollow(
 			// implicit continue
 		}
 	}
+
+	return exclusive
 }
 
 // timelineStatusUpdate streams the edited status to the user using the