From 6607e1c9444d0814b72762a46814ff0812d96343 Mon Sep 17 00:00:00 2001
From: kim
Date: Thu, 18 Sep 2025 16:33:23 +0200
Subject: [feature] add paging support to rss feed endpoint, and support JSON /
atom feed types (#4442)
originally based on: https://codeberg.org/superseriousbusiness/gotosocial/pulls/4396
hope this is okay https://codeberg.org/zordsdavini !
closes https://codeberg.org/superseriousbusiness/gotosocial/issues/4411
closes https://codeberg.org/superseriousbusiness/gotosocial/issues/3407
Co-authored-by: Arnas Udovic
Reviewed-on: https://codeberg.org/superseriousbusiness/gotosocial/pulls/4442
Co-authored-by: kim
Co-committed-by: kim
---
internal/processing/account/rss.go | 73 +++-----
internal/processing/account/rss_test.go | 289 +++++++++++++++++++++++++++++---
internal/processing/account/statuses.go | 11 +-
3 files changed, 299 insertions(+), 74 deletions(-)
(limited to 'internal/processing/account')
diff --git a/internal/processing/account/rss.go b/internal/processing/account/rss.go
index 495aa2e54..205027528 100644
--- a/internal/processing/account/rss.go
+++ b/internal/processing/account/rss.go
@@ -20,21 +20,19 @@ package account
import (
"context"
"errors"
- "fmt"
"time"
"code.superseriousbusiness.org/gotosocial/internal/config"
"code.superseriousbusiness.org/gotosocial/internal/db"
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
+ "code.superseriousbusiness.org/gotosocial/internal/paging"
"github.com/gorilla/feeds"
)
-const (
- rssFeedLength = 20
-)
+var never time.Time
-type GetRSSFeed func() (string, gtserror.WithCode)
+type GetRSSFeed func() (*feeds.Feed, gtserror.WithCode)
// GetRSSFeedForUsername returns a function to return the RSS feed of a local account
// with the given username, and the last-modified time (time that the account last
@@ -45,33 +43,30 @@ type GetRSSFeed func() (string, gtserror.WithCode)
//
// If the account has not yet posted an RSS-eligible status, the returned last-modified
// time will be zero, and the GetRSSFeed func will return a valid RSS xml with no items.
-func (p *Processor) GetRSSFeedForUsername(ctx context.Context, username string) (GetRSSFeed, time.Time, gtserror.WithCode) {
- var (
- never = time.Time{}
- )
+func (p *Processor) GetRSSFeedForUsername(ctx context.Context, username string, page *paging.Page) (GetRSSFeed, time.Time, gtserror.WithCode) {
+ // Fetch local (i.e. empty domain) account from database by username.
account, err := p.state.DB.GetAccountByUsernameDomain(ctx, username, "")
if err != nil {
- if errors.Is(err, db.ErrNoEntries) {
- // Simply no account with this username.
- err = gtserror.New("account not found")
- return nil, never, gtserror.NewErrorNotFound(err)
- }
-
- // Real db error.
- err = gtserror.Newf("db error getting account %s: %w", username, err)
+ err := gtserror.Newf("db error getting account %s: %w", username, err)
return nil, never, gtserror.NewErrorInternalError(err)
}
+ // Check if exists.
+ if account == nil {
+ err := gtserror.New("account not found")
+ return nil, never, gtserror.NewErrorNotFound(err)
+ }
+
// Ensure account has rss feed enabled.
if !*account.Settings.EnableRSS {
- err = gtserror.New("account RSS feed not enabled")
+ err := gtserror.New("account RSS feed not enabled")
return nil, never, gtserror.NewErrorNotFound(err)
}
- // Ensure account stats populated.
+ // Ensure account stats populated for last status fetch information.
if err := p.state.DB.PopulateAccountStats(ctx, account); err != nil {
- err = gtserror.Newf("db error getting account stats %s: %w", username, err)
+ err := gtserror.Newf("db error getting account stats %s: %w", username, err)
return nil, never, gtserror.NewErrorInternalError(err)
}
@@ -80,14 +75,14 @@ func (p *Processor) GetRSSFeedForUsername(ctx context.Context, username string)
// eligible to appear in the RSS feed; that's fine.
lastPostAt := account.Stats.LastStatusAt
- return func() (string, gtserror.WithCode) {
+ return func() (*feeds.Feed, gtserror.WithCode) {
// Assemble author namestring once only.
author := "@" + account.Username + "@" + config.GetAccountDomain()
- // Derive image/thumbnail for this account (may be nil).
+ // Derive image/thumbnail for this account (may be nil if no media).
image, errWithCode := p.rssImageForAccount(ctx, account, author)
if errWithCode != nil {
- return "", errWithCode
+ return nil, errWithCode
}
feed := &feeds.Feed{
@@ -106,7 +101,7 @@ func (p *Processor) GetRSSFeedForUsername(ctx context.Context, username string)
// since we already know there's no eligible statuses.
if lastPostAt.IsZero() {
feed.Updated = account.CreatedAt
- return stringifyFeed(feed)
+ return feed, nil
}
// Account has posted at least one status that's
@@ -120,32 +115,30 @@ func (p *Processor) GetRSSFeedForUsername(ctx context.Context, username string)
//
// Take into account whether the user wants
// their web view laid out in gallery mode.
- mediaOnly := account.Settings != nil &&
- account.Settings.WebLayout == gtsmodel.WebLayoutGallery
+ mediaOnly := (account.Settings != nil &&
+ account.Settings.WebLayout == gtsmodel.WebLayoutGallery)
statuses, err := p.state.DB.GetAccountWebStatuses(
ctx,
account,
+ page,
mediaOnly,
- rssFeedLength,
- "", // Latest posts from the top.
)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
- err = fmt.Errorf("db error getting account web statuses: %w", err)
- return "", gtserror.NewErrorInternalError(err)
+ err := gtserror.Newf("db error getting account web statuses: %w", err)
+ return nil, gtserror.NewErrorInternalError(err)
}
// Add each status to the rss feed.
for _, status := range statuses {
item, err := p.converter.StatusToRSSItem(ctx, status)
if err != nil {
- err = gtserror.Newf("error converting status to feed item: %w", err)
- return "", gtserror.NewErrorInternalError(err)
+ err := gtserror.Newf("error converting status to feed item: %w", err)
+ return nil, gtserror.NewErrorInternalError(err)
}
-
feed.Add(item)
}
- return stringifyFeed(feed)
+ return feed, nil
}, lastPostAt, nil
}
@@ -177,15 +170,3 @@ func (p *Processor) rssImageForAccount(ctx context.Context, account *gtsmodel.Ac
Link: account.URL,
}, nil
}
-
-func stringifyFeed(feed *feeds.Feed) (string, gtserror.WithCode) {
- // Stringify the feed. Even with no statuses,
- // this will still produce valid rss xml.
- rss, err := feed.ToRss()
- if err != nil {
- err := gtserror.Newf("error converting feed to rss string: %w", err)
- return "", gtserror.NewErrorInternalError(err)
- }
-
- return rss, nil
-}
diff --git a/internal/processing/account/rss_test.go b/internal/processing/account/rss_test.go
index 0b64e8464..b053a3795 100644
--- a/internal/processing/account/rss_test.go
+++ b/internal/processing/account/rss_test.go
@@ -19,7 +19,10 @@ package account_test
import (
"testing"
+ "time"
+ "code.superseriousbusiness.org/gotosocial/internal/paging"
+ "github.com/gorilla/feeds"
"github.com/stretchr/testify/suite"
)
@@ -28,13 +31,8 @@ type GetRSSTestSuite struct {
}
func (suite *GetRSSTestSuite) TestGetAccountRSSAdmin() {
- getFeed, lastModified, err := suite.accountProcessor.GetRSSFeedForUsername(suite.T().Context(), "admin")
- suite.NoError(err)
- suite.EqualValues(1634726497, lastModified.Unix())
-
- feed, err := getFeed()
- suite.NoError(err)
- suite.Equal(`
+ suite.testGetFeedSerializedAs("admin", &paging.Page{Limit: 20}, (*feeds.Feed).ToRss, 1634726497,
+ `Posts from @admin@localhost:8080
http://localhost:8080/@admin
@@ -63,17 +61,177 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSAdmin() {
http://localhost:8080/@admin/feed.rss
-`, feed)
+`)
+}
+
+func (suite *GetRSSTestSuite) TestGetAccountAtomAdmin() {
+ suite.testGetFeedSerializedAs("admin", &paging.Page{Limit: 20}, (*feeds.Feed).ToAtom, 1634726497,
+ `
+ Posts from @admin@localhost:8080
+ http://localhost:8080/@admin
+ 2021-10-20T10:41:37Z
+ Posts from @admin@localhost:8080
+
+
+ open to see some <strong>puppies</strong>
+ 2021-10-20T12:36:45Z
+ http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37
+ <p>🐕🐕🐕🐕🐕</p>
+
+
+ @admin@localhost:8080 made a new post: "🐕🐕🐕🐕🐕"
+
+ @admin@localhost:8080
+
+
+
+ hello world! #welcome ! first post on the instance :rainbow: !
+ 2021-10-20T11:36:45Z
+ http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R
+ <p>hello world! <a href="http://localhost:8080/tags/welcome" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>welcome</span></a> ! first post on the instance <img src="http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png" title=":rainbow:" alt=":rainbow:" width="25" height="25" /> !</p>
+
+
+ @admin@localhost:8080 posted 1 attachment: "hello world! #welcome ! first post on the instance :rainbow: !"
+
+ @admin@localhost:8080
+
+
+`)
+}
+
+func (suite *GetRSSTestSuite) TestGetAccountJSONAdmin() {
+ suite.testGetFeedSerializedAs("admin", &paging.Page{Limit: 20}, (*feeds.Feed).ToJSON, 1634726497,
+ `{
+ "version": "https://jsonfeed.org/version/1.1",
+ "title": "Posts from @admin@localhost:8080",
+ "home_page_url": "http://localhost:8080/@admin",
+ "description": "Posts from @admin@localhost:8080",
+ "items": [
+ {
+ "id": "http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37",
+ "url": "http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37",
+ "external_url": "http://localhost:8080/@admin/feed.rss",
+ "title": "open to see some \u003cstrong\u003epuppies\u003c/strong\u003e",
+ "content_html": "\u003cp\u003e🐕🐕🐕🐕🐕\u003c/p\u003e",
+ "summary": "@admin@localhost:8080 made a new post: \"🐕🐕🐕🐕🐕\"",
+ "date_published": "2021-10-20T12:36:45Z",
+ "author": {
+ "name": "@admin@localhost:8080"
+ },
+ "authors": [
+ {
+ "name": "@admin@localhost:8080"
+ }
+ ]
+ },
+ {
+ "id": "http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R",
+ "url": "http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R",
+ "external_url": "http://localhost:8080/@admin/feed.rss",
+ "title": "hello world! #welcome ! first post on the instance :rainbow: !",
+ "content_html": "\u003cp\u003ehello world! \u003ca href=\"http://localhost:8080/tags/welcome\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\"\u003e#\u003cspan\u003ewelcome\u003c/span\u003e\u003c/a\u003e ! first post on the instance \u003cimg src=\"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png\" title=\":rainbow:\" alt=\":rainbow:\" width=\"25\" height=\"25\" /\u003e !\u003c/p\u003e",
+ "summary": "@admin@localhost:8080 posted 1 attachment: \"hello world! #welcome ! first post on the instance :rainbow: !\"",
+ "image": "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg",
+ "date_published": "2021-10-20T11:36:45Z",
+ "author": {
+ "name": "@admin@localhost:8080"
+ },
+ "authors": [
+ {
+ "name": "@admin@localhost:8080"
+ }
+ ]
+ }
+ ]
+}`)
}
func (suite *GetRSSTestSuite) TestGetAccountRSSZork() {
- getFeed, lastModified, err := suite.accountProcessor.GetRSSFeedForUsername(suite.T().Context(), "the_mighty_zork")
- suite.NoError(err)
- suite.EqualValues(1730451600, lastModified.Unix())
+ suite.testGetFeedSerializedAs("the_mighty_zork", &paging.Page{Limit: 20}, (*feeds.Feed).ToRss, 1730451600,
+ `
+
+ Posts from @the_mighty_zork@localhost:8080
+ http://localhost:8080/@the_mighty_zork
+ Posts from @the_mighty_zork@localhost:8080
+ Fri, 01 Nov 2024 09:00:00 +0000
+ Fri, 01 Nov 2024 09:00:00 +0000
+
+ http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.webp
+ Avatar for @the_mighty_zork@localhost:8080
+ http://localhost:8080/@the_mighty_zork
+
+
+ edited status
+ http://localhost:8080/@the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR
+ @the_mighty_zork@localhost:8080 made a new post: "this is the latest revision of the status, with a content-warning"
+ this is the latest revision of the status, with a content-warning
]]>
+ @the_mighty_zork@localhost:8080
+ http://localhost:8080/@the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR
+ Fri, 01 Nov 2024 09:00:00 +0000
+ http://localhost:8080/@the_mighty_zork/feed.rss
+
+
+ HTML in post
+ http://localhost:8080/@the_mighty_zork/statuses/01HH9KYNQPA416TNJ53NSATP40
+ @the_mighty_zork@localhost:8080 made a new post: "Here's a bunch of HTML, read it and weep, weep then!
`+"```"+`html
<section class="about-user">
<div class="col-header">
<h2>About</h2>
</div>
<div class="fields">
<h3 class="sr-only">Fields</h3>
<dl>
...
+ Here's a bunch of HTML, read it and weep, weep then!