From 3cceed11b28b5f42a653d85ed779d652fd8c26ad Mon Sep 17 00:00:00 2001 From: tobi <31960611+tsmethurst@users.noreply.github.com> Date: Tue, 16 Apr 2024 13:10:13 +0200 Subject: [feature/performance] Store account stats in separate table (#2831) * [feature/performance] Store account stats in separate table, get stats from remote * test account stats * add some missing increment / decrement calls * change stats function signatures * rejig logging a bit * use lock when updating stats --- internal/processing/account/delete.go | 5 +++++ internal/processing/account/move.go | 2 +- internal/processing/account/rss.go | 14 +++++++++----- internal/processing/account/rss_test.go | 14 ++------------ 4 files changed, 17 insertions(+), 18 deletions(-) (limited to 'internal/processing/account') diff --git a/internal/processing/account/delete.go b/internal/processing/account/delete.go index 858e42d36..a900c566d 100644 --- a/internal/processing/account/delete.go +++ b/internal/processing/account/delete.go @@ -485,6 +485,11 @@ func (p *Processor) deleteAccountPeripheral(ctx context.Context, account *gtsmod return gtserror.Newf("error deleting poll votes by account: %w", err) } + // Delete account stats model. + if err := p.state.DB.DeleteAccountStats(ctx, account.ID); err != nil { + return gtserror.Newf("error deleting stats for account: %w", err) + } + return nil } diff --git a/internal/processing/account/move.go b/internal/processing/account/move.go index a68c8f750..602e8c021 100644 --- a/internal/processing/account/move.go +++ b/internal/processing/account/move.go @@ -113,7 +113,7 @@ func (p *Processor) MoveSelf( // in quick succession, so get a lock on // this account. lockKey := originAcct.URI - unlock := p.state.ClientLocks.Lock(lockKey) + unlock := p.state.AccountLocks.Lock(lockKey) defer unlock() // Ensure we have a valid, up-to-date representation of the target account. diff --git a/internal/processing/account/rss.go b/internal/processing/account/rss.go index f2c6cba5e..cbbb4875b 100644 --- a/internal/processing/account/rss.go +++ b/internal/processing/account/rss.go @@ -69,14 +69,18 @@ func (p *Processor) GetRSSFeedForUsername(ctx context.Context, username string) return nil, never, gtserror.NewErrorNotFound(err) } + // Ensure account stats populated. + if account.Stats == nil { + if err := p.state.DB.PopulateAccountStats(ctx, account); err != nil { + err = gtserror.Newf("db error getting account stats %s: %w", username, err) + return nil, never, gtserror.NewErrorInternalError(err) + } + } + // LastModified time is needed by callers to check freshness for cacheing. // This might be a zero time.Time if account has never posted a status that's // eligible to appear in the RSS feed; that's fine. - lastPostAt, err := p.state.DB.GetAccountLastPosted(ctx, account.ID, true) - if err != nil && !errors.Is(err, db.ErrNoEntries) { - err = gtserror.Newf("db error getting account %s last posted: %w", username, err) - return nil, never, gtserror.NewErrorInternalError(err) - } + lastPostAt := account.Stats.LastStatusAt return func() (string, gtserror.WithCode) { // Assemble author namestring once only. diff --git a/internal/processing/account/rss_test.go b/internal/processing/account/rss_test.go index 6ae285f9e..c08ef8874 100644 --- a/internal/processing/account/rss_test.go +++ b/internal/processing/account/rss_test.go @@ -19,7 +19,6 @@ package account_test import ( "context" - "fmt" "testing" "github.com/stretchr/testify/suite" @@ -32,14 +31,11 @@ type GetRSSTestSuite struct { func (suite *GetRSSTestSuite) TestGetAccountRSSAdmin() { getFeed, lastModified, err := suite.accountProcessor.GetRSSFeedForUsername(context.Background(), "admin") suite.NoError(err) - suite.EqualValues(1634733405, lastModified.Unix()) + suite.EqualValues(1634726497, lastModified.Unix()) feed, err := getFeed() suite.NoError(err) - - fmt.Println(feed) - - suite.Equal("\n \n Posts from @admin@localhost:8080\n http://localhost:8080/@admin\n Posts from @admin@localhost:8080\n Wed, 20 Oct 2021 12:36:45 +0000\n Wed, 20 Oct 2021 12:36:45 +0000\n \n open to see some puppies\n http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37\n @admin@localhost:8080 made a new post: "🐕🐕🐕🐕🐕"\n \n @admin@localhost:8080\n http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37\n Wed, 20 Oct 2021 12:36:45 +0000\n http://localhost:8080/@admin/feed.rss\n \n \n hello world! #welcome ! first post on the instance :rainbow: !\n http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R\n @admin@localhost:8080 posted 1 attachment: "hello world! #welcome ! first post on the instance :rainbow: !"\n !]]>\n @admin@localhost:8080\n \n http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R\n Wed, 20 Oct 2021 11:36:45 +0000\n http://localhost:8080/@admin/feed.rss\n \n \n", feed) + suite.Equal("\n \n Posts from @admin@localhost:8080\n http://localhost:8080/@admin\n Posts from @admin@localhost:8080\n Wed, 20 Oct 2021 10:41:37 +0000\n Wed, 20 Oct 2021 10:41:37 +0000\n \n open to see some puppies\n http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37\n @admin@localhost:8080 made a new post: "🐕🐕🐕🐕🐕"\n \n @admin@localhost:8080\n http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37\n Wed, 20 Oct 2021 12:36:45 +0000\n http://localhost:8080/@admin/feed.rss\n \n \n hello world! #welcome ! first post on the instance :rainbow: !\n http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R\n @admin@localhost:8080 posted 1 attachment: "hello world! #welcome ! first post on the instance :rainbow: !"\n !]]>\n @admin@localhost:8080\n \n http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R\n Wed, 20 Oct 2021 11:36:45 +0000\n http://localhost:8080/@admin/feed.rss\n \n \n", feed) } func (suite *GetRSSTestSuite) TestGetAccountRSSZork() { @@ -49,9 +45,6 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSZork() { feed, err := getFeed() suite.NoError(err) - - fmt.Println(feed) - suite.Equal("\n \n Posts from @the_mighty_zork@localhost:8080\n http://localhost:8080/@the_mighty_zork\n Posts from @the_mighty_zork@localhost:8080\n Sun, 10 Dec 2023 09:24:00 +0000\n Sun, 10 Dec 2023 09:24:00 +0000\n \n http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpg\n Avatar for @the_mighty_zork@localhost:8080\n http://localhost:8080/@the_mighty_zork\n \n \n HTML in post\n http://localhost:8080/@the_mighty_zork/statuses/01HH9KYNQPA416TNJ53NSATP40\n @the_mighty_zork@localhost:8080 made a new post: "Here's a bunch of HTML, read it and weep, weep then! ```html <section class="about-user"> <div class="col-header"> <h2>About</h2> </div> <div class="fields"> <h3 class="sr-only">Fields</h3> <dl> ...\n Here's a bunch of HTML, read it and weep, weep then!

<section class="about-user">\n    <div class="col-header">\n        <h2>About</h2>\n    </div>            \n    <div class="fields">\n        <h3 class="sr-only">Fields</h3>\n        <dl>\n            <div class="field">\n                <dt>should you follow me?</dt>\n                <dd>maybe!</dd>\n            </div>\n            <div class="field">\n                <dt>age</dt>\n                <dd>120</dd>\n            </div>\n        </dl>\n    </div>\n    <div class="bio">\n        <h3 class="sr-only">Bio</h3>\n        <p>i post about things that concern me</p>\n    </div>\n    <div class="sr-only" role="group">\n        <h3 class="sr-only">Stats</h3>\n        <span>Joined in Jun, 2022.</span>\n        <span>8 posts.</span>\n        <span>Followed by 1.</span>\n        <span>Following 1.</span>\n    </div>\n    <div class="accountstats" aria-hidden="true">\n        <b>Joined</b><time datetime="2022-06-04T13:12:00.000Z">Jun, 2022</time>\n        <b>Posts</b><span>8</span>\n        <b>Followed by</b><span>1</span>\n        <b>Following</b><span>1</span>\n    </div>\n</section>\n

There, hope you liked that!

]]>\n @the_mighty_zork@localhost:8080\n http://localhost:8080/@the_mighty_zork/statuses/01HH9KYNQPA416TNJ53NSATP40\n Sun, 10 Dec 2023 09:24:00 +0000\n http://localhost:8080/@the_mighty_zork/feed.rss\n \n \n introduction post\n http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY\n @the_mighty_zork@localhost:8080 made a new post: "hello everyone!"\n \n @the_mighty_zork@localhost:8080\n http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY\n Wed, 20 Oct 2021 10:40:37 +0000\n http://localhost:8080/@the_mighty_zork/feed.rss\n \n \n", feed) } @@ -77,9 +70,6 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSZorkNoPosts() { feed, err := getFeed() suite.NoError(err) - - fmt.Println(feed) - suite.Equal("\n \n Posts from @the_mighty_zork@localhost:8080\n http://localhost:8080/@the_mighty_zork\n Posts from @the_mighty_zork@localhost:8080\n Fri, 20 May 2022 11:09:18 +0000\n Fri, 20 May 2022 11:09:18 +0000\n \n http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpg\n Avatar for @the_mighty_zork@localhost:8080\n http://localhost:8080/@the_mighty_zork\n \n \n", feed) } -- cgit v1.2.3