[feature] Add paging via `Link` header for notifications and account statuses (#629)

* test link headers

* page get account statuses properly

* page get notifications

* add util func for packaging timeline responses

* return timelined stuff from accountstatusesget

* rename timeline response

* use new convenience function

* go fmt

7 files changed, 109 insertions, 84 deletions

diff --git a/internal/processing/account.go b/internal/processing/account.go
index 25f024785..43c4f65e6 100644
--- a/internal/processing/account.go
+++ b/internal/processing/account.go
@@ -46,7 +46,7 @@ func (p *processor) AccountUpdate(ctx context.Context, authed *oauth.Auth, form
 	return p.accountProcessor.Update(ctx, authed.Account, form)
 }
 
-func (p *processor) AccountStatusesGet(ctx context.Context, authed *oauth.Auth, targetAccountID string, limit int, excludeReplies bool, excludeReblogs bool, maxID string, minID string, pinnedOnly bool, mediaOnly bool, publicOnly bool) ([]apimodel.Status, gtserror.WithCode) {
+func (p *processor) AccountStatusesGet(ctx context.Context, authed *oauth.Auth, targetAccountID string, limit int, excludeReplies bool, excludeReblogs bool, maxID string, minID string, pinnedOnly bool, mediaOnly bool, publicOnly bool) (*apimodel.TimelineResponse, gtserror.WithCode) {
 	return p.accountProcessor.StatusesGet(ctx, authed.Account, targetAccountID, limit, excludeReplies, excludeReblogs, maxID, minID, pinnedOnly, mediaOnly, publicOnly)
 }
 
diff --git a/internal/processing/account/account.go b/internal/processing/account/account.go
index 7668da02c..2a27fc743 100644
--- a/internal/processing/account/account.go
+++ b/internal/processing/account/account.go
@@ -55,7 +55,7 @@ type Processor interface {
 	Update(ctx context.Context, account *gtsmodel.Account, form *apimodel.UpdateCredentialsRequest) (*apimodel.Account, error)
 	// StatusesGet fetches a number of statuses (in time descending order) from the given account, filtered by visibility for
 	// the account given in authed.
-	StatusesGet(ctx context.Context, requestingAccount *gtsmodel.Account, targetAccountID string, limit int, excludeReplies bool, excludeReblogs bool, maxID string, minID string, pinned bool, mediaOnly bool, publicOnly bool) ([]apimodel.Status, gtserror.WithCode)
+	StatusesGet(ctx context.Context, requestingAccount *gtsmodel.Account, targetAccountID string, limit int, excludeReplies bool, excludeReblogs bool, maxID string, minID string, pinned bool, mediaOnly bool, publicOnly bool) (*apimodel.TimelineResponse, gtserror.WithCode)
 	// FollowersGet fetches a list of the target account's followers.
 	FollowersGet(ctx context.Context, requestingAccount *gtsmodel.Account, targetAccountID string) ([]apimodel.Account, gtserror.WithCode)
 	// FollowingGet fetches a list of the accounts that target account is following.
diff --git a/internal/processing/account/getstatuses.go b/internal/processing/account/getstatuses.go
index c185302c5..90bf8e06e 100644
--- a/internal/processing/account/getstatuses.go
+++ b/internal/processing/account/getstatuses.go
@@ -26,9 +26,11 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/timeline"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
-func (p *processor) StatusesGet(ctx context.Context, requestingAccount *gtsmodel.Account, targetAccountID string, limit int, excludeReplies bool, excludeReblogs bool, maxID string, minID string, pinnedOnly bool, mediaOnly bool, publicOnly bool) ([]apimodel.Status, gtserror.WithCode) {
+func (p *processor) StatusesGet(ctx context.Context, requestingAccount *gtsmodel.Account, targetAccountID string, limit int, excludeReplies bool, excludeReblogs bool, maxID string, minID string, pinnedOnly bool, mediaOnly bool, publicOnly bool) (*apimodel.TimelineResponse, gtserror.WithCode) {
 	if requestingAccount != nil {
 		if blocked, err := p.db.IsBlocked(ctx, requestingAccount.ID, targetAccountID, true); err != nil {
 			return nil, gtserror.NewErrorInternalError(err)
@@ -37,29 +39,48 @@ func (p *processor) StatusesGet(ctx context.Context, requestingAccount *gtsmodel
 		}
 	}
 
-	apiStatuses := []apimodel.Status{}
-
 	statuses, err := p.db.GetAccountStatuses(ctx, targetAccountID, limit, excludeReplies, excludeReblogs, maxID, minID, pinnedOnly, mediaOnly, publicOnly)
 	if err != nil {
 		if err == db.ErrNoEntries {
-			return apiStatuses, nil
+			return util.EmptyTimelineResponse(), nil
 		}
 		return nil, gtserror.NewErrorInternalError(err)
 	}
 
+	var filtered []*gtsmodel.Status
 	for _, s := range statuses {
 		visible, err := p.filter.StatusVisible(ctx, s, requestingAccount)
-		if err != nil || !visible {
-			continue
+		if err == nil && visible {
+			filtered = append(filtered, s)
 		}
+	}
+
+	if len(filtered) == 0 {
+		return util.EmptyTimelineResponse(), nil
+	}
 
-		apiStatus, err := p.tc.StatusToAPIStatus(ctx, s, requestingAccount)
+	timelineables := []timeline.Timelineable{}
+	for _, i := range filtered {
+		apiStatus, err := p.tc.StatusToAPIStatus(ctx, i, requestingAccount)
 		if err != nil {
 			return nil, gtserror.NewErrorInternalError(fmt.Errorf("error converting status to api: %s", err))
 		}
 
-		apiStatuses = append(apiStatuses, *apiStatus)
+		timelineables = append(timelineables, apiStatus)
 	}
 
-	return apiStatuses, nil
+	return util.PackageTimelineableResponse(util.TimelineableResponseParams{
+		Items:          timelineables,
+		Path:           fmt.Sprintf("/api/v1/accounts/%s/statuses", targetAccountID),
+		NextMaxIDValue: timelineables[len(timelineables)-1].GetID(),
+		PrevMinIDValue: timelineables[0].GetID(),
+		Limit:          limit,
+		ExtraQueryParams: []string{
+			fmt.Sprintf("exclude_replies=%t", excludeReplies),
+			fmt.Sprintf("exclude_reblogs=%t", excludeReblogs),
+			fmt.Sprintf("pinned_only=%t", pinnedOnly),
+			fmt.Sprintf("only_media=%t", mediaOnly),
+			fmt.Sprintf("only_public=%t", publicOnly),
+		},
+	})
 }
diff --git a/internal/processing/notification.go b/internal/processing/notification.go
index e976952e8..ee0e4ae34 100644
--- a/internal/processing/notification.go
+++ b/internal/processing/notification.go
@@ -26,9 +26,11 @@ import (
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+	"github.com/superseriousbusiness/gotosocial/internal/timeline"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
-func (p *processor) NotificationsGet(ctx context.Context, authed *oauth.Auth, limit int, maxID string, sinceID string) ([]*apimodel.Notification, gtserror.WithCode) {
+func (p *processor) NotificationsGet(ctx context.Context, authed *oauth.Auth, limit int, maxID string, sinceID string) (*apimodel.TimelineResponse, gtserror.WithCode) {
 	l := logrus.WithField("func", "NotificationsGet")
 
 	notifs, err := p.db.GetNotifications(ctx, authed.Account.ID, limit, maxID, sinceID)
@@ -36,15 +38,26 @@ func (p *processor) NotificationsGet(ctx context.Context, authed *oauth.Auth, li
 		return nil, gtserror.NewErrorInternalError(err)
 	}
 
-	apiNotifs := []*apimodel.Notification{}
+	if len(notifs) == 0 {
+		return util.EmptyTimelineResponse(), nil
+	}
+
+	timelineables := []timeline.Timelineable{}
 	for _, n := range notifs {
 		apiNotif, err := p.tc.NotificationToAPINotification(ctx, n)
 		if err != nil {
 			l.Debugf("got an error converting a notification to api, will skip it: %s", err)
 			continue
 		}
-		apiNotifs = append(apiNotifs, apiNotif)
+		timelineables = append(timelineables, apiNotif)
 	}
 
-	return apiNotifs, nil
+	return util.PackageTimelineableResponse(util.TimelineableResponseParams{
+		Items:          timelineables,
+		Path:           "api/v1/notifications",
+		NextMaxIDValue: timelineables[len(timelineables)-1].GetID(),
+		PrevMinIDKey:   "since_id",
+		PrevMinIDValue: timelineables[0].GetID(),
+		Limit:          limit,
+	})
 }
diff --git a/internal/processing/notification_test.go b/internal/processing/notification_test.go
index 6f2d44c5c..6ecca92a9 100644
--- a/internal/processing/notification_test.go
+++ b/internal/processing/notification_test.go
@@ -23,6 +23,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/suite"
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 )
 
 type NotificationTestSuite struct {
@@ -32,14 +33,19 @@ type NotificationTestSuite struct {
 // get a notification where someone has liked our status
 func (suite *NotificationTestSuite) TestGetNotifications() {
 	receivingAccount := suite.testAccounts["local_account_1"]
-	notifs, err := suite.processor.NotificationsGet(context.Background(), suite.testAutheds["local_account_1"], 10, "", "")
+	notifsResponse, err := suite.processor.NotificationsGet(context.Background(), suite.testAutheds["local_account_1"], 10, "", "")
 	suite.NoError(err)
-	suite.Len(notifs, 1)
-	notif := notifs[0]
+	suite.Len(notifsResponse.Items, 1)
+	notif, ok := notifsResponse.Items[0].(*apimodel.Notification)
+	if !ok {
+		panic("notif in response wasn't *apimodel.Notification")
+	}
+
 	suite.NotNil(notif.Status)
 	suite.NotNil(notif.Status)
 	suite.NotNil(notif.Status.Account)
 	suite.Equal(receivingAccount.ID, notif.Status.Account.ID)
+	suite.Equal(`<http://localhost:8080/api/v1/notifications?limit=10&max_id=01F8Q0ANPTWW10DAKTX7BRPBJP>; rel="next", <http://localhost:8080/api/v1/notifications?limit=10&since_id=01F8Q0ANPTWW10DAKTX7BRPBJP>; rel="prev"`, notifsResponse.LinkHeader)
 }
 
 func TestNotificationTestSuite(t *testing.T) {
diff --git a/internal/processing/processor.go b/internal/processing/processor.go
index 225606b70..a10e0d6b3 100644
--- a/internal/processing/processor.go
+++ b/internal/processing/processor.go
@@ -83,7 +83,7 @@ type Processor interface {
 	AccountUpdate(ctx context.Context, authed *oauth.Auth, form *apimodel.UpdateCredentialsRequest) (*apimodel.Account, error)
 	// AccountStatusesGet fetches a number of statuses (in time descending order) from the given account, filtered by visibility for
 	// the account given in authed.
-	AccountStatusesGet(ctx context.Context, authed *oauth.Auth, targetAccountID string, limit int, excludeReplies bool, excludeReblogs bool, maxID string, minID string, pinned bool, mediaOnly bool, publicOnly bool) ([]apimodel.Status, gtserror.WithCode)
+	AccountStatusesGet(ctx context.Context, authed *oauth.Auth, targetAccountID string, limit int, excludeReplies bool, excludeReblogs bool, maxID string, minID string, pinned bool, mediaOnly bool, publicOnly bool) (*apimodel.TimelineResponse, gtserror.WithCode)
 	// AccountFollowersGet fetches a list of the target account's followers.
 	AccountFollowersGet(ctx context.Context, authed *oauth.Auth, targetAccountID string) ([]apimodel.Account, gtserror.WithCode)
 	// AccountFollowingGet fetches a list of the accounts that target account is following.
@@ -150,7 +150,7 @@ type Processor interface {
 	MediaUpdate(ctx context.Context, authed *oauth.Auth, attachmentID string, form *apimodel.AttachmentUpdateRequest) (*apimodel.Attachment, gtserror.WithCode)
 
 	// NotificationsGet
-	NotificationsGet(ctx context.Context, authed *oauth.Auth, limit int, maxID string, sinceID string) ([]*apimodel.Notification, gtserror.WithCode)
+	NotificationsGet(ctx context.Context, authed *oauth.Auth, limit int, maxID string, sinceID string) (*apimodel.TimelineResponse, gtserror.WithCode)
 
 	// SearchGet performs a search with the given params, resolving/dereferencing remotely as desired
 	SearchGet(ctx context.Context, authed *oauth.Auth, searchQuery *apimodel.SearchQuery) (*apimodel.SearchResult, gtserror.WithCode)
@@ -177,11 +177,11 @@ type Processor interface {
 	StatusGetContext(ctx context.Context, authed *oauth.Auth, targetStatusID string) (*apimodel.Context, gtserror.WithCode)
 
 	// HomeTimelineGet returns statuses from the home timeline, with the given filters/parameters.
-	HomeTimelineGet(ctx context.Context, authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) (*apimodel.StatusTimelineResponse, gtserror.WithCode)
+	HomeTimelineGet(ctx context.Context, authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) (*apimodel.TimelineResponse, gtserror.WithCode)
 	// PublicTimelineGet returns statuses from the public/local timeline, with the given filters/parameters.
-	PublicTimelineGet(ctx context.Context, authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) (*apimodel.StatusTimelineResponse, gtserror.WithCode)
+	PublicTimelineGet(ctx context.Context, authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) (*apimodel.TimelineResponse, gtserror.WithCode)
 	// FavedTimelineGet returns faved statuses, with the given filters/parameters.
-	FavedTimelineGet(ctx context.Context, authed *oauth.Auth, maxID string, minID string, limit int) (*apimodel.StatusTimelineResponse, gtserror.WithCode)
+	FavedTimelineGet(ctx context.Context, authed *oauth.Auth, maxID string, minID string, limit int) (*apimodel.TimelineResponse, gtserror.WithCode)
 
 	// AuthorizeStreamingRequest returns a gotosocial account in exchange for an access token, or an error if the given token is not valid.
 	AuthorizeStreamingRequest(ctx context.Context, accessToken string) (*gtsmodel.Account, error)
diff --git a/internal/processing/statustimeline.go b/internal/processing/statustimeline.go
index d4388f381..081709302 100644
--- a/internal/processing/statustimeline.go
+++ b/internal/processing/statustimeline.go
@@ -22,18 +22,17 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"net/url"
 
 	"github.com/sirupsen/logrus"
 
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
-	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 	"github.com/superseriousbusiness/gotosocial/internal/timeline"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
 	"github.com/superseriousbusiness/gotosocial/internal/visibility"
 )
 
@@ -139,114 +138,100 @@ func StatusSkipInsertFunction() timeline.SkipInsertFunction {
 	}
 }
 
-func (p *processor) packageStatusResponse(statuses []*apimodel.Status, path string, nextMaxID string, prevMinID string, limit int) (*apimodel.StatusTimelineResponse, gtserror.WithCode) {
-	resp := &apimodel.StatusTimelineResponse{
-		Statuses: []*apimodel.Status{},
-	}
-	resp.Statuses = statuses
-
-	// prepare the next and previous links
-	if len(statuses) != 0 {
-		protocol := config.GetProtocol()
-		host := config.GetHost()
-
-		nextLink := &url.URL{
-			Scheme:   protocol,
-			Host:     host,
-			Path:     path,
-			RawQuery: fmt.Sprintf("limit=%d&max_id=%s", limit, nextMaxID),
-		}
-		next := fmt.Sprintf("<%s>; rel=\"next\"", nextLink.String())
-
-		prevLink := &url.URL{
-			Scheme:   protocol,
-			Host:     host,
-			Path:     path,
-			RawQuery: fmt.Sprintf("limit=%d&min_id=%s", limit, prevMinID),
-		}
-		prev := fmt.Sprintf("<%s>; rel=\"prev\"", prevLink.String())
-		resp.LinkHeader = fmt.Sprintf("%s, %s", next, prev)
-	}
-
-	return resp, nil
-}
-
-func (p *processor) HomeTimelineGet(ctx context.Context, authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) (*apimodel.StatusTimelineResponse, gtserror.WithCode) {
+func (p *processor) HomeTimelineGet(ctx context.Context, authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) (*apimodel.TimelineResponse, gtserror.WithCode) {
 	preparedItems, err := p.statusTimelines.GetTimeline(ctx, authed.Account.ID, maxID, sinceID, minID, limit, local)
 	if err != nil {
 		return nil, gtserror.NewErrorInternalError(err)
 	}
 
 	if len(preparedItems) == 0 {
-		return &apimodel.StatusTimelineResponse{
-			Statuses: []*apimodel.Status{},
-		}, nil
+		return util.EmptyTimelineResponse(), nil
 	}
 
-	statuses := []*apimodel.Status{}
+	timelineables := []timeline.Timelineable{}
 	for _, i := range preparedItems {
 		status, ok := i.(*apimodel.Status)
 		if !ok {
 			return nil, gtserror.NewErrorInternalError(errors.New("error converting prepared timeline entry to api status"))
 		}
-		statuses = append(statuses, status)
+		timelineables = append(timelineables, status)
 	}
 
-	return p.packageStatusResponse(statuses, "api/v1/timelines/home", statuses[len(preparedItems)-1].ID, statuses[0].ID, limit)
+	return util.PackageTimelineableResponse(util.TimelineableResponseParams{
+		Items:          timelineables,
+		Path:           "api/v1/timelines/home",
+		NextMaxIDValue: timelineables[len(timelineables)-1].GetID(),
+		PrevMinIDValue: timelineables[0].GetID(),
+		Limit:          limit,
+	})
 }
 
-func (p *processor) PublicTimelineGet(ctx context.Context, authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) (*apimodel.StatusTimelineResponse, gtserror.WithCode) {
+func (p *processor) PublicTimelineGet(ctx context.Context, authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) (*apimodel.TimelineResponse, gtserror.WithCode) {
 	statuses, err := p.db.GetPublicTimeline(ctx, authed.Account.ID, maxID, sinceID, minID, limit, local)
 	if err != nil {
 		if err == db.ErrNoEntries {
 			// there are just no entries left
-			return &apimodel.StatusTimelineResponse{
-				Statuses: []*apimodel.Status{},
-			}, nil
+			return util.EmptyTimelineResponse(), nil
 		}
 		// there's an actual error
 		return nil, gtserror.NewErrorInternalError(err)
 	}
 
-	s, err := p.filterPublicStatuses(ctx, authed, statuses)
+	filtered, err := p.filterPublicStatuses(ctx, authed, statuses)
 	if err != nil {
 		return nil, gtserror.NewErrorInternalError(err)
 	}
 
-	if len(s) == 0 {
-		return &apimodel.StatusTimelineResponse{
-			Statuses: []*apimodel.Status{},
-		}, nil
+	if len(filtered) == 0 {
+		return util.EmptyTimelineResponse(), nil
+	}
+
+	timelineables := []timeline.Timelineable{}
+	for _, i := range filtered {
+		timelineables = append(timelineables, i)
 	}
 
-	return p.packageStatusResponse(s, "api/v1/timelines/public", s[len(s)-1].ID, s[0].ID, limit)
+	return util.PackageTimelineableResponse(util.TimelineableResponseParams{
+		Items:          timelineables,
+		Path:           "api/v1/timelines/public",
+		NextMaxIDValue: timelineables[len(timelineables)-1].GetID(),
+		PrevMinIDValue: timelineables[0].GetID(),
+		Limit:          limit,
+	})
 }
 
-func (p *processor) FavedTimelineGet(ctx context.Context, authed *oauth.Auth, maxID string, minID string, limit int) (*apimodel.StatusTimelineResponse, gtserror.WithCode) {
+func (p *processor) FavedTimelineGet(ctx context.Context, authed *oauth.Auth, maxID string, minID string, limit int) (*apimodel.TimelineResponse, gtserror.WithCode) {
 	statuses, nextMaxID, prevMinID, err := p.db.GetFavedTimeline(ctx, authed.Account.ID, maxID, minID, limit)
 	if err != nil {
 		if err == db.ErrNoEntries {
 			// there are just no entries left
-			return &apimodel.StatusTimelineResponse{
-				Statuses: []*apimodel.Status{},
-			}, nil
+			return util.EmptyTimelineResponse(), nil
 		}
 		// there's an actual error
 		return nil, gtserror.NewErrorInternalError(err)
 	}
 
-	s, err := p.filterFavedStatuses(ctx, authed, statuses)
+	filtered, err := p.filterFavedStatuses(ctx, authed, statuses)
 	if err != nil {
 		return nil, gtserror.NewErrorInternalError(err)
 	}
 
-	if len(s) == 0 {
-		return &apimodel.StatusTimelineResponse{
-			Statuses: []*apimodel.Status{},
-		}, nil
+	if len(filtered) == 0 {
+		return util.EmptyTimelineResponse(), nil
+	}
+
+	timelineables := []timeline.Timelineable{}
+	for _, i := range filtered {
+		timelineables = append(timelineables, i)
 	}
 
-	return p.packageStatusResponse(s, "api/v1/favourites", nextMaxID, prevMinID, limit)
+	return util.PackageTimelineableResponse(util.TimelineableResponseParams{
+		Items:          timelineables,
+		Path:           "api/v1/favourites",
+		NextMaxIDValue: nextMaxID,
+		PrevMinIDValue: prevMinID,
+		Limit:          limit,
+	})
 }
 
 func (p *processor) filterPublicStatuses(ctx context.Context, authed *oauth.Auth, statuses []*gtsmodel.Status) ([]*apimodel.Status, error) {