diff options
Diffstat (limited to 'internal')
| -rw-r--r-- | internal/api/client/admin/domainpermissionsubscriptioncreate.go | 11 | ||||
| -rw-r--r-- | internal/api/client/admin/domainpermissionsubscriptionupdate.go | 11 | ||||
| -rw-r--r-- | internal/api/model/domain.go | 7 | ||||
| -rw-r--r-- | internal/db/bundb/domain.go | 28 | ||||
| -rw-r--r-- | internal/db/bundb/migrations/20250603125942_domain_perm_sub_retractions.go | 76 | ||||
| -rw-r--r-- | internal/db/bundb/migrations/20250603125942_domain_perm_sub_retractions/domainpermissionsubscription.go | 42 | ||||
| -rw-r--r-- | internal/db/domain.go | 6 | ||||
| -rw-r--r-- | internal/gtsmodel/adminaction.go | 5 | ||||
| -rw-r--r-- | internal/gtsmodel/domainpermissionsubscription.go | 90 | ||||
| -rw-r--r-- | internal/processing/admin/domainallow.go | 2 | ||||
| -rw-r--r-- | internal/processing/admin/domainpermissionsubscription.go | 13 | ||||
| -rw-r--r-- | internal/subscriptions/domainperms.go | 216 | ||||
| -rw-r--r-- | internal/subscriptions/subscriptions_test.go | 132 | ||||
| -rw-r--r-- | internal/typeutils/internaltofrontend.go | 1 |
14 files changed, 608 insertions, 32 deletions
diff --git a/internal/api/client/admin/domainpermissionsubscriptioncreate.go b/internal/api/client/admin/domainpermissionsubscriptioncreate.go index f44eda748..e57b4bc3f 100644 --- a/internal/api/client/admin/domainpermissionsubscriptioncreate.go +++ b/internal/api/client/admin/domainpermissionsubscriptioncreate.go @@ -94,6 +94,15 @@ import ( // type: boolean // default: false // - +// name: remove_retracted +// in: formData +// description: >- +// If true, then when a list is processed, if the list does *not* contain entries that +// it *did* contain previously, ie., retracted entries, then domain permissions +// corresponding to those entries will be removed. If false, they will just be orphaned instead. +// type: boolean +// default: true +// - // name: uri // required: true // in: formData @@ -234,6 +243,8 @@ func (m *Module) DomainPermissionSubscriptionPOSTHandler(c *gin.Context) { contentType, permType, asDraft, + form.AdoptOrphans, + form.RemoveRetracted, util.PtrOrZero(form.FetchUsername), // Optional. util.PtrOrZero(form.FetchPassword), // Optional. ) diff --git a/internal/api/client/admin/domainpermissionsubscriptionupdate.go b/internal/api/client/admin/domainpermissionsubscriptionupdate.go index 364e75f80..416f8ec10 100644 --- a/internal/api/client/admin/domainpermissionsubscriptionupdate.go +++ b/internal/api/client/admin/domainpermissionsubscriptionupdate.go @@ -97,6 +97,15 @@ import ( // type: boolean // default: false // - +// name: remove_retracted +// in: formData +// description: >- +// If true, then when a list is processed, if the list does *not* contain entries that +// it *did* contain previously, ie., retracted entries, then domain permissions +// corresponding to those entries will be removed. If false, they will just be orphaned instead. +// type: boolean +// default: true +// - // name: content_type // in: formData // description: >- @@ -227,6 +236,7 @@ func (m *Module) DomainPermissionSubscriptionPATCHHandler(c *gin.Context) { contentType == nil && form.AsDraft == nil && form.AdoptOrphans == nil && + form.RemoveRetracted == nil && form.FetchUsername == nil && form.FetchPassword == nil { const errText = "no updateable fields set on request" @@ -244,6 +254,7 @@ func (m *Module) DomainPermissionSubscriptionPATCHHandler(c *gin.Context) { contentType, form.AsDraft, form.AdoptOrphans, + form.RemoveRetracted, form.FetchUsername, form.FetchPassword, ) diff --git a/internal/api/model/domain.go b/internal/api/model/domain.go index b793d77b0..c6d868975 100644 --- a/internal/api/model/domain.go +++ b/internal/api/model/domain.go @@ -131,6 +131,9 @@ type DomainPermissionSubscription struct { // If true, this domain permission subscription will "adopt" domain permissions which already exist on the instance, and which meet the following conditions: 1) they have no subscription ID (ie., they're "orphaned") and 2) they are present in the subscribed list. Such orphaned domain permissions will be given this subscription's subscription ID value. // example: false AdoptOrphans bool `json:"adopt_orphans"` + // If true, then when a list is processed, if the list does *not* contain entries that it *did* contain previously, ie., retracted entries, then domain permissions corresponding to those entries will be removed. If false, they will just be orphaned instead. + // example: true + RemoveRetracted bool `json:"remove_retracted"` // Time at which the subscription was created (ISO 8601 Datetime). // example: 2021-07-30T09:20:25+00:00 CreatedAt string `json:"created_at"` @@ -198,6 +201,10 @@ type DomainPermissionSubscriptionRequest struct { // in the subscribed list. Such orphaned domain permissions will be given this // subscription's subscription ID value and be managed by this subscription. AdoptOrphans *bool `form:"adopt_orphans" json:"adopt_orphans"` + // If true, then when a list is processed, if the list does *not* contain entries that + // it *did* contain previously, ie., retracted entries, then domain permissions + // corresponding to those entries will be removed. If false, they will just be orphaned instead. + RemoveRetracted *bool `form:"remove_retracted" json:"remove_retracted"` // (Optional) username to set for basic auth when doing a fetch of URI. // example: admin123 FetchUsername *string `form:"fetch_username" json:"fetch_username"` diff --git a/internal/db/bundb/domain.go b/internal/db/bundb/domain.go index 3d9ab41d0..4e3058de1 100644 --- a/internal/db/bundb/domain.go +++ b/internal/db/bundb/domain.go @@ -97,6 +97,20 @@ func (d *domainDB) GetDomainAllows(ctx context.Context) ([]*gtsmodel.DomainAllow return allows, nil } +func (d *domainDB) GetDomainAllowsBySubscriptionID(ctx context.Context, subscriptionID string) ([]*gtsmodel.DomainAllow, error) { + allows := []*gtsmodel.DomainAllow{} + + if err := d.db. + NewSelect(). + Model(&allows). + Where("? = ?", bun.Ident("subscription_id"), subscriptionID). + Scan(ctx); err != nil { + return nil, err + } + + return allows, nil +} + func (d *domainDB) GetDomainAllowByID(ctx context.Context, id string) (*gtsmodel.DomainAllow, error) { var allow gtsmodel.DomainAllow @@ -225,6 +239,20 @@ func (d *domainDB) GetDomainBlocks(ctx context.Context) ([]*gtsmodel.DomainBlock return blocks, nil } +func (d *domainDB) GetDomainBlocksBySubscriptionID(ctx context.Context, subscriptionID string) ([]*gtsmodel.DomainBlock, error) { + blocks := []*gtsmodel.DomainBlock{} + + if err := d.db. + NewSelect(). + Model(&blocks). + Where("? = ?", bun.Ident("subscription_id"), subscriptionID). + Scan(ctx); err != nil { + return nil, err + } + + return blocks, nil +} + func (d *domainDB) GetDomainBlockByID(ctx context.Context, id string) (*gtsmodel.DomainBlock, error) { var block gtsmodel.DomainBlock diff --git a/internal/db/bundb/migrations/20250603125942_domain_perm_sub_retractions.go b/internal/db/bundb/migrations/20250603125942_domain_perm_sub_retractions.go new file mode 100644 index 000000000..83237a614 --- /dev/null +++ b/internal/db/bundb/migrations/20250603125942_domain_perm_sub_retractions.go @@ -0,0 +1,76 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package migrations + +import ( + "context" + "fmt" + "reflect" + + "code.superseriousbusiness.org/gotosocial/internal/db/bundb/migrations/20250603125942_domain_perm_sub_retractions" + "code.superseriousbusiness.org/gotosocial/internal/log" + "github.com/uptrace/bun" +) + +func init() { + up := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + // Bail if "remove_retracted" + // column already created. + if exists, err := doesColumnExist( + ctx, + tx, + "domain_permission_subscriptions", + "remove_retracted", + ); err != nil { + return err + } else if exists { + return nil + } + + // Derive column definition. + var permSub *gtsmodel.DomainPermissionSubscription + permSubType := reflect.TypeOf(permSub) + colDef, err := getBunColumnDef(tx, permSubType, "RemoveRetracted") + if err != nil { + return fmt.Errorf("error making column def: %w", err) + } + + log.Info(ctx, "adding domain_permission_subscriptions.remove_retracted column...") + if _, err := tx. + NewAddColumn(). + Model(permSub). + ColumnExpr(colDef). + Exec(ctx); err != nil { + return fmt.Errorf("error adding column: %w", err) + } + + return nil + }) + } + + down := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + return nil + }) + } + + if err := Migrations.Register(up, down); err != nil { + panic(err) + } +} diff --git a/internal/db/bundb/migrations/20250603125942_domain_perm_sub_retractions/domainpermissionsubscription.go b/internal/db/bundb/migrations/20250603125942_domain_perm_sub_retractions/domainpermissionsubscription.go new file mode 100644 index 000000000..a8d00cb06 --- /dev/null +++ b/internal/db/bundb/migrations/20250603125942_domain_perm_sub_retractions/domainpermissionsubscription.go @@ -0,0 +1,42 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package gtsmodel + +import "time" + +type DomainPermissionSubscription struct { + ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` + Priority uint8 `bun:""` + Title string `bun:",nullzero,unique"` + PermissionType uint8 `bun:",nullzero,notnull"` + AsDraft *bool `bun:",nullzero,notnull,default:true"` + AdoptOrphans *bool `bun:",nullzero,notnull,default:false"` + CreatedByAccountID string `bun:"type:CHAR(26),nullzero,notnull"` + URI string `bun:",nullzero,notnull,unique"` + ContentType int16 `bun:",nullzero,notnull"` + FetchUsername string `bun:",nullzero"` + FetchPassword string `bun:",nullzero"` + FetchedAt time.Time `bun:"type:timestamptz,nullzero"` + SuccessfullyFetchedAt time.Time `bun:"type:timestamptz,nullzero"` + LastModified time.Time `bun:"type:timestamptz,nullzero"` + ETag string `bun:"etag,nullzero"` + Error string `bun:",nullzero"` + + // This is the field added by this migration. + RemoveRetracted *bool `bun:",nullzero,notnull,default:true"` +} diff --git a/internal/db/domain.go b/internal/db/domain.go index 0c1fb97b3..466fb0a4f 100644 --- a/internal/db/domain.go +++ b/internal/db/domain.go @@ -43,6 +43,9 @@ type Domain interface { // GetDomainAllows returns all instance-level domain allows currently enforced by this instance. GetDomainAllows(ctx context.Context) ([]*gtsmodel.DomainAllow, error) + // GetDomainAllowsBySubscriptionID gets all domain allows that have the given subscription ID. + GetDomainAllowsBySubscriptionID(ctx context.Context, subscriptionID string) ([]*gtsmodel.DomainAllow, error) + // UpdateDomainAllow updates the given domain allow, setting the provided columns (empty for all). UpdateDomainAllow(ctx context.Context, allow *gtsmodel.DomainAllow, columns ...string) error @@ -58,6 +61,9 @@ type Domain interface { // GetDomainBlockByID returns one instance-level domain block with the given id, if it exists. GetDomainBlockByID(ctx context.Context, id string) (*gtsmodel.DomainBlock, error) + // GetDomainBlocksBySubscriptionID gets all domain blocks that have the given subscription ID. + GetDomainBlocksBySubscriptionID(ctx context.Context, subscriptionID string) ([]*gtsmodel.DomainBlock, error) + // GetDomainBlocks returns all instance-level domain blocks currently enforced by this instance. GetDomainBlocks(ctx context.Context) ([]*gtsmodel.DomainBlock, error) diff --git a/internal/gtsmodel/adminaction.go b/internal/gtsmodel/adminaction.go index 5ca8244a0..6374379ce 100644 --- a/internal/gtsmodel/adminaction.go +++ b/internal/gtsmodel/adminaction.go @@ -74,6 +74,7 @@ const ( AdminActionSuspend AdminActionUnsuspend AdminActionExpireKeys + AdminActionUnallow ) func (t AdminActionType) String() string { @@ -92,6 +93,8 @@ func (t AdminActionType) String() string { return "unsuspend" case AdminActionExpireKeys: return "expire-keys" + case AdminActionUnallow: + return "unallow" default: return "unknown" } @@ -113,6 +116,8 @@ func ParseAdminActionType(in string) AdminActionType { return AdminActionUnsuspend case "expire-keys": return AdminActionExpireKeys + case "unallow": + return AdminActionUnallow default: return AdminActionUnknown } diff --git a/internal/gtsmodel/domainpermissionsubscription.go b/internal/gtsmodel/domainpermissionsubscription.go index 724b2164f..2a27fc140 100644 --- a/internal/gtsmodel/domainpermissionsubscription.go +++ b/internal/gtsmodel/domainpermissionsubscription.go @@ -20,23 +20,79 @@ package gtsmodel import "time" type DomainPermissionSubscription struct { - ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // ID of this item in the database. - Priority uint8 `bun:""` // Priority of this subscription compared to others of the same permission type. 0-255 (higher = higher priority). - Title string `bun:",nullzero,unique"` // Moderator-set title for this list. - PermissionType DomainPermissionType `bun:",nullzero,notnull"` // Permission type of the subscription. - AsDraft *bool `bun:",nullzero,notnull,default:true"` // Create domain permission entries resulting from this subscription as drafts. - AdoptOrphans *bool `bun:",nullzero,notnull,default:false"` // Adopt orphaned domain permissions present in this subscription's entries. - CreatedByAccountID string `bun:"type:CHAR(26),nullzero,notnull"` // Account ID of the creator of this subscription. - CreatedByAccount *Account `bun:"-"` // Account corresponding to createdByAccountID. - URI string `bun:",nullzero,notnull,unique"` // URI of the domain permission list. - ContentType DomainPermSubContentType `bun:",nullzero,notnull"` // Content type to expect from the URI. - FetchUsername string `bun:",nullzero"` // Username to send when doing a GET of URI using basic auth. - FetchPassword string `bun:",nullzero"` // Password to send when doing a GET of URI using basic auth. - FetchedAt time.Time `bun:"type:timestamptz,nullzero"` // Time when fetch of URI was last attempted. - SuccessfullyFetchedAt time.Time `bun:"type:timestamptz,nullzero"` // Time when the domain permission list was last *successfuly* fetched, to be transmitted as If-Modified-Since header. - LastModified time.Time `bun:"type:timestamptz,nullzero"` // "Last-Modified" time received from the server (if any) on last successful fetch. Used for HTTP request caching. - ETag string `bun:"etag,nullzero"` // "ETag" header last received from the server (if any) on last successful fetch. Used for HTTP request caching. - Error string `bun:",nullzero"` // If latest fetch attempt errored, this field stores the error message. Cleared on latest successful fetch. + // ID of this item in the database. + ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` + + // Priority of this subscription compared + // to others of the same permission type. + // 0-255 (higher = higher priority). + Priority uint8 `bun:""` + + // Moderator-set title for this list. + Title string `bun:",nullzero,unique"` + + // Permission type of the subscription. + PermissionType DomainPermissionType `bun:",nullzero,notnull"` + + // Create domain permission entries + // resulting from this subscription as drafts. + AsDraft *bool `bun:",nullzero,notnull,default:true"` + + // Adopt orphaned domain permissions + // present in this subscription's entries. + AdoptOrphans *bool `bun:",nullzero,notnull,default:false"` + + // Account ID of the creator of this subscription. + CreatedByAccountID string `bun:"type:CHAR(26),nullzero,notnull"` + + // Account corresponding to createdByAccountID. + CreatedByAccount *Account `bun:"-"` + + // URI of the domain permission list. + URI string `bun:",nullzero,notnull,unique"` + + // Content type to expect from the URI. + ContentType DomainPermSubContentType `bun:",nullzero,notnull"` + + // Username to send when doing + // a GET of URI using basic auth. + FetchUsername string `bun:",nullzero"` + + // Password to send when doing + // a GET of URI using basic auth. + FetchPassword string `bun:",nullzero"` + + // Time when fetch of URI was last attempted. + FetchedAt time.Time `bun:"type:timestamptz,nullzero"` + + // Time when the domain permission list + // was last *successfuly* fetched, to be + // transmitted as If-Modified-Since header. + SuccessfullyFetchedAt time.Time `bun:"type:timestamptz,nullzero"` + + // "Last-Modified" time received from the + // server (if any) on last successful fetch. + // Used for HTTP request caching. + LastModified time.Time `bun:"type:timestamptz,nullzero"` + + // "ETag" header last received from the + // server (if any) on last successful fetch. + // Used for HTTP request caching. + ETag string `bun:"etag,nullzero"` + + // If latest fetch attempt errored, + // this field stores the error message. + // Cleared on latest successful fetch. + Error string `bun:",nullzero"` + + // If true, then when a list is processed, if the + // list does *not* contain entries that it *did* + // contain previously, ie., retracted entries, + // then domain permissions corresponding to those + // entries will be removed. + // + // If false, they will just be orphaned instead. + RemoveRetracted *bool `bun:",nullzero,notnull,default:true"` } type DomainPermSubContentType enumType diff --git a/internal/processing/admin/domainallow.go b/internal/processing/admin/domainallow.go index 388ca5cb1..bdf70642b 100644 --- a/internal/processing/admin/domainallow.go +++ b/internal/processing/admin/domainallow.go @@ -176,7 +176,7 @@ func (p *Processor) deleteDomainAllow( ID: id.NewULID(), TargetCategory: gtsmodel.AdminActionCategoryDomain, TargetID: domainAllow.Domain, - Type: gtsmodel.AdminActionUnsuspend, + Type: gtsmodel.AdminActionUnallow, AccountID: adminAcct.ID, } diff --git a/internal/processing/admin/domainpermissionsubscription.go b/internal/processing/admin/domainpermissionsubscription.go index b4dc72aa3..9afe6ee5c 100644 --- a/internal/processing/admin/domainpermissionsubscription.go +++ b/internal/processing/admin/domainpermissionsubscription.go @@ -142,6 +142,8 @@ func (p *Processor) DomainPermissionSubscriptionCreate( contentType gtsmodel.DomainPermSubContentType, permType gtsmodel.DomainPermissionType, asDraft bool, + adoptOrphans *bool, + removeRetracted *bool, fetchUsername string, fetchPassword string, ) (*apimodel.DomainPermissionSubscription, gtserror.WithCode) { @@ -151,12 +153,14 @@ func (p *Processor) DomainPermissionSubscriptionCreate( Title: title, PermissionType: permType, AsDraft: &asDraft, + AdoptOrphans: adoptOrphans, CreatedByAccountID: acct.ID, CreatedByAccount: acct, URI: uri, ContentType: contentType, FetchUsername: fetchUsername, FetchPassword: fetchPassword, + RemoveRetracted: removeRetracted, } err := p.state.DB.PutDomainPermissionSubscription(ctx, permSub) @@ -184,6 +188,7 @@ func (p *Processor) DomainPermissionSubscriptionUpdate( contentType *gtsmodel.DomainPermSubContentType, asDraft *bool, adoptOrphans *bool, + removeRetracted *bool, fetchUsername *string, fetchPassword *string, ) (*apimodel.DomainPermissionSubscription, gtserror.WithCode) { @@ -230,6 +235,11 @@ func (p *Processor) DomainPermissionSubscriptionUpdate( columns = append(columns, "adopt_orphans") } + if removeRetracted != nil { + permSub.RemoveRetracted = removeRetracted + columns = append(columns, "remove_retracted") + } + if fetchPassword != nil { permSub.FetchPassword = *fetchPassword columns = append(columns, "fetch_password") @@ -342,12 +352,13 @@ func (p *Processor) DomainPermissionSubscriptionTest( // Call the permSub.URI and parse a list of perms from it. // Any error returned here is a "real" one, not an error // from fetching / parsing the list. - createdPerms, err := p.subscriptions.ProcessDomainPermissionSubscription( + createdPerms, _, err := p.subscriptions.ProcessDomainPermissionSubscription( ctx, permSub, tsport, higherPrios, true, // Dry run. + true, // Skip caching. ) if err != nil { err := gtserror.Newf("error doing dry-run: %w", err) diff --git a/internal/subscriptions/domainperms.go b/internal/subscriptions/domainperms.go index 1572ffb19..0f001620f 100644 --- a/internal/subscriptions/domainperms.go +++ b/internal/subscriptions/domainperms.go @@ -166,17 +166,19 @@ func (s *Subscriptions) ProcessDomainPermissionSubscriptions( return } + var skipCache bool for i, permSub := range permSubs { // Higher priority permission subs = everything // above this permission sub in the slice. higherPrios := permSubs[:i] - _, err := s.ProcessDomainPermissionSubscription( + _, retracted, err := s.ProcessDomainPermissionSubscription( ctx, permSub, tsport, higherPrios, - false, // Not dry. Wet, if you will. + false, // Not dry. Wet, if you will. + skipCache, // Skip cache if necessary. ) if err != nil { // Real db error. @@ -187,6 +189,16 @@ func (s *Subscriptions) ProcessDomainPermissionSubscriptions( return } + // If any retractions have been done, skip caching + // when doing subsequent fetches. This makes it so + // that if an entry was present in a higher-priority + // list and a lower-priority list, but was retracted + // from the higher-priority list, it will be created + // and managed by the lower-priority list instead. + if retracted && !skipCache { + skipCache = true + } + // Update this perm sub. err = s.state.DB.UpdateDomainPermissionSubscription(ctx, permSub) if err != nil { @@ -203,7 +215,10 @@ func (s *Subscriptions) ProcessDomainPermissionSubscriptions( // entry in the database, or ignoring it if it's excluded or already // covered by a higher-priority subscription. // -// On success, the slice of discovered DomainPermissions will be returned. +// On success, the slice of discovered DomainPermissions will be returned, +// including a boolean to indicate whether or not any retractions have been +// performed since the list was last checked (if ever). +// // In case of parsing error, or error on the remote side, permSub.Error // will be updated with the calling/parsing error, and `nil, nil` will be // returned. In case of an actual db error, `nil, err` will be returned and @@ -215,6 +230,10 @@ func (s *Subscriptions) ProcessDomainPermissionSubscriptions( // If dry == true, then the URI will still be called, and permissions // will be parsed, but they will not actually be created. // +// If skipCache == true, then conditional HTTP request headers will not be +// sent, and so cached values for the domain permission subscription list +// will not be used (ie., the list will always be fetched "fresh"). +// // Note that while this function modifies fields on the given permSub, // it's up to the caller to update it in the database (if desired). func (s *Subscriptions) ProcessDomainPermissionSubscription( @@ -223,7 +242,8 @@ func (s *Subscriptions) ProcessDomainPermissionSubscription( tsport transport.Transport, higherPrios []*gtsmodel.DomainPermissionSubscription, dry bool, -) ([]gtsmodel.DomainPermission, error) { + skipCache bool, +) ([]gtsmodel.DomainPermission, bool, error) { l := log. WithContext(ctx). WithFields(kv.Fields{ @@ -235,10 +255,10 @@ func (s *Subscriptions) ProcessDomainPermissionSubscription( // going to attempt this now. permSub.FetchedAt = time.Now() - // Call the URI, and only skip - // cache if we're doing a dry run. + // Call the URI, skipping conditional requests + // (caching) if we've been told to do so. resp, err := tsport.DereferenceDomainPermissions( - ctx, permSub, dry, + ctx, permSub, skipCache, ) if err != nil { // Couldn't get this one, @@ -246,7 +266,7 @@ func (s *Subscriptions) ProcessDomainPermissionSubscription( errStr := err.Error() l.Warnf("couldn't dereference permSubURI: %+v", err) permSub.Error = errStr - return nil, nil + return nil, false, nil } // If the permissions at URI weren't modified @@ -257,7 +277,7 @@ func (s *Subscriptions) ProcessDomainPermissionSubscription( permSub.ETag = resp.ETag permSub.LastModified = resp.LastModified permSub.SuccessfullyFetchedAt = permSub.FetchedAt - return nil, nil + return nil, false, nil } // At this point we know we got a 200 OK @@ -289,7 +309,7 @@ func (s *Subscriptions) ProcessDomainPermissionSubscription( errStr := err.Error() l.Warnf("couldn't parse results: %+v", err) permSub.Error = errStr - return nil, nil + return nil, false, nil } if len(wantedPerms) == 0 { @@ -299,7 +319,7 @@ func (s *Subscriptions) ProcessDomainPermissionSubscription( const errStr = "fetch successful but parsed zero usable results" l.Warn(errStr) permSub.Error = errStr - return nil, nil + return nil, false, nil } // This can now be considered a successful fetch. @@ -325,7 +345,7 @@ func (s *Subscriptions) ProcessDomainPermissionSubscription( ) if err != nil { // Proper db error. - return nil, err + return nil, false, err } if !created { @@ -335,7 +355,30 @@ func (s *Subscriptions) ProcessDomainPermissionSubscription( createdPerms = append(createdPerms, wantedPerm) } - return createdPerms, nil + if dry { + // Don't do any further + // processing with a dry run. + return createdPerms, false, nil + } + + // Process any retractions since + // the last time list was checked. + // + // Being unable to do retractions + // isn't the end of the world as it + // can be tried again next time the + // list is updated, so if there was + // an error just warn it. + retracted, err := s.processRetractions( + ctx, l, + permSub, + wantedPerms, + ) + if err != nil { + l.Warnf("error doing retractions: %+v", err) + } + + return createdPerms, retracted, nil } // processDomainPermission processes one wanted domain @@ -895,3 +938,150 @@ func (s *Subscriptions) adoptPerm( return err } + +func (s *Subscriptions) processRetractions( + ctx context.Context, + l log.Entry, + permSub *gtsmodel.DomainPermissionSubscription, + wantedPerms []gtsmodel.DomainPermission, +) (bool, error) { + var ( + isBlocks = permSub.PermissionType == gtsmodel.DomainPermissionBlock + removeRetracted = *permSub.RemoveRetracted + + // True if at least one + // retraction has occurred. + retracted bool + ) + + // Gather existing perms into an interface type. + existingPerms := []gtsmodel.DomainPermission{} + if isBlocks { + existingBlocks, err := s.state.DB.GetDomainBlocksBySubscriptionID(ctx, permSub.ID) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + // Proper db error. + err := gtserror.Newf("db error getting existing blocks owned by perm sub: %w", err) + return retracted, err + } + for _, existingBlock := range existingBlocks { + existingPerms = append(existingPerms, existingBlock) + } + } else { + existingAllows, err := s.state.DB.GetDomainAllowsBySubscriptionID(ctx, permSub.ID) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + // Proper db error. + err := gtserror.Newf("db error getting existing allows owned by perm sub: %w", err) + return retracted, err + } + for _, existingAllow := range existingAllows { + existingPerms = append(existingPerms, existingAllow) + } + } + + // For each existing permission, check if + // it's included in the list of wanted perms + // that's just been freshly fetched + created. + // + // If it's not, we should consider it retracted + // and handle retraction effects appropriately. + for _, existingPerm := range existingPerms { + if slices.ContainsFunc( + wantedPerms, + func(wantedPerm gtsmodel.DomainPermission) bool { + return existingPerm.GetDomain() == wantedPerm.GetDomain() + }, + ) { + // This permission from the + // database exists in wanted + // perms, so it's not been + // retracted, leave it alone. + continue + } + + // This perm exists in the database but + // not in wanted perms, so it has been + // retracted, check what we need to do. + domain := existingPerm.GetDomain() + l.WithField("domain", domain).Info("handling retraction") + + var ( + dbF func() error + action *gtsmodel.AdminAction + actionF admin.ActionF + ) + + switch { + + // Remove this block. + case isBlocks && removeRetracted: + dbF = func() error { return s.state.DB.DeleteDomainBlock(ctx, domain) } + action = >smodel.AdminAction{ + ID: id.NewULID(), + TargetCategory: gtsmodel.AdminActionCategoryDomain, + TargetID: domain, + Type: gtsmodel.AdminActionUnsuspend, + AccountID: permSub.CreatedByAccountID, + } + actionF = s.state.AdminActions.DomainUnblockF( + action.ID, + existingPerm.(*gtsmodel.DomainBlock), + ) + + // Remove this allow. + case !isBlocks && removeRetracted: + dbF = func() error { return s.state.DB.DeleteDomainAllow(ctx, domain) } + action = >smodel.AdminAction{ + ID: id.NewULID(), + TargetCategory: gtsmodel.AdminActionCategoryDomain, + TargetID: domain, + Type: gtsmodel.AdminActionUnallow, + AccountID: permSub.CreatedByAccountID, + } + actionF = s.state.AdminActions.DomainUnallowF( + action.ID, + existingPerm.(*gtsmodel.DomainAllow), + ) + + // Orphan this block. + case isBlocks: + block := existingPerm.(*gtsmodel.DomainBlock) + block.SubscriptionID = "" + dbF = func() error { return s.state.DB.UpdateDomainBlock(ctx, block, "subscription_id") } + + // Orphan this allow. + case !isBlocks: + allow := existingPerm.(*gtsmodel.DomainAllow) + allow.SubscriptionID = "" + dbF = func() error { return s.state.DB.UpdateDomainAllow(ctx, allow, "subscription_id") } + } + + // Run the retraction db + // func to either delete + // or update the perm. + if err := dbF(); err != nil { + return retracted, err + } + + // Mark that at least one + // retraction has been done. + if !retracted { + retracted = true + } + + if action == nil { + // No side effects; + // nothing else to do. + continue + } + + // Run the side effects. + if err := s.state.AdminActions.Run(ctx, action, actionF); err != nil { + err := gtserror.Newf("error running side effects: %w", err) + return retracted, err + } + + // TODO: Remove draft(s) as well? + } + + return retracted, nil +} diff --git a/internal/subscriptions/subscriptions_test.go b/internal/subscriptions/subscriptions_test.go index 8e292209d..193c073f0 100644 --- a/internal/subscriptions/subscriptions_test.go +++ b/internal/subscriptions/subscriptions_test.go @@ -949,6 +949,138 @@ func (suite *SubscriptionsTestSuite) TestDomainAllowsAndBlocks() { suite.Equal(existingAllow.SubscriptionID, testAllowSubscription.ID) } +func (suite *SubscriptionsTestSuite) TestRemoveRetraction() { + var ( + ctx = suite.T().Context() + testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath) + testAccount = suite.testAccounts["admin_account"] + subscriptions = subscriptions.New( + testStructs.State, + testStructs.TransportController, + testStructs.TypeConverter, + ) + + // A subscription for a plain list of + // baddies, which removes retracted entries. + testSubscription = >smodel.DomainPermissionSubscription{ + ID: "01JGE681TQSBPAV59GZXPKE62H", + Priority: 255, + Title: "whatever!", + PermissionType: gtsmodel.DomainPermissionBlock, + AsDraft: util.Ptr(false), + AdoptOrphans: util.Ptr(false), + CreatedByAccountID: testAccount.ID, + CreatedByAccount: testAccount, + URI: "https://lists.example.org/baddies.txt", + ContentType: gtsmodel.DomainPermSubContentTypePlain, + RemoveRetracted: util.Ptr(true), + } + + // Block owned by testSubscription + // that no longer exists on the remote + // list, ie., it's been retracted. + retractedBlock = >smodel.DomainBlock{ + ID: "01JHX2V5WN250TKB6FQ1M3QE1H", + Domain: "retracted.example.org", + CreatedByAccount: testAccount, + CreatedByAccountID: testAccount.ID, + SubscriptionID: "01JGE681TQSBPAV59GZXPKE62H", + } + ) + defer testrig.TearDownTestStructs(testStructs) + + // Store test subscription. + if err := testStructs.State.DB.PutDomainPermissionSubscription( + ctx, testSubscription, + ); err != nil { + suite.FailNow(err.Error()) + } + + // Store the retracted block. + if err := testStructs.State.DB.PutDomainBlock( + ctx, retractedBlock, + ); err != nil { + suite.FailNow(err.Error()) + } + + // Process subscriptions. + subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType) + + // Retracted block should be removed. + if !testrig.WaitFor(func() bool { + _, err := testStructs.State.DB.GetDomainBlock(ctx, retractedBlock.Domain) + return errors.Is(err, db.ErrNoEntries) + }) { + suite.FailNow("timed out waiting for block to be removed") + } +} + +func (suite *SubscriptionsTestSuite) TestOrphanRetraction() { + var ( + ctx = suite.T().Context() + testStructs = testrig.SetupTestStructs(rMediaPath, rTemplatePath) + testAccount = suite.testAccounts["admin_account"] + subscriptions = subscriptions.New( + testStructs.State, + testStructs.TransportController, + testStructs.TypeConverter, + ) + + // A subscription for a plain list of + // baddies, which orphans retracted entries. + testSubscription = >smodel.DomainPermissionSubscription{ + ID: "01JGE681TQSBPAV59GZXPKE62H", + Priority: 255, + Title: "whatever!", + PermissionType: gtsmodel.DomainPermissionBlock, + AsDraft: util.Ptr(false), + AdoptOrphans: util.Ptr(false), + CreatedByAccountID: testAccount.ID, + CreatedByAccount: testAccount, + URI: "https://lists.example.org/baddies.txt", + ContentType: gtsmodel.DomainPermSubContentTypePlain, + RemoveRetracted: util.Ptr(false), + } + + // Block owned by testSubscription + // that no longer exists on the remote + // list, ie., it's been retracted. + retractedBlock = >smodel.DomainBlock{ + ID: "01JHX2V5WN250TKB6FQ1M3QE1H", + Domain: "retracted.example.org", + CreatedByAccount: testAccount, + CreatedByAccountID: testAccount.ID, + SubscriptionID: "01JGE681TQSBPAV59GZXPKE62H", + } + ) + defer testrig.TearDownTestStructs(testStructs) + + // Store test subscription. + if err := testStructs.State.DB.PutDomainPermissionSubscription( + ctx, testSubscription, + ); err != nil { + suite.FailNow(err.Error()) + } + + // Store the retracted block. + if err := testStructs.State.DB.PutDomainBlock( + ctx, retractedBlock, + ); err != nil { + suite.FailNow(err.Error()) + } + + // Process subscriptions. + subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType) + + // Retracted block should be orphaned. + if !testrig.WaitFor(func() bool { + block, err := testStructs.State.DB.GetDomainBlock(ctx, retractedBlock.Domain) + return err == nil && block.SubscriptionID == "" + }) { + suite.FailNow("timed out waiting for block to be orphaned") + } +} + func TestSubscriptionTestSuite(t *testing.T) { suite.Run(t, new(SubscriptionsTestSuite)) } diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index bdb33243d..3ac7e1536 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -2212,6 +2212,7 @@ func (c *Converter) DomainPermSubToAPIDomainPermSub( PermissionType: d.PermissionType.String(), AsDraft: *d.AsDraft, AdoptOrphans: *d.AdoptOrphans, + RemoveRetracted: *d.RemoveRetracted, CreatedBy: d.CreatedByAccountID, CreatedAt: util.FormatISO8601(createdAt), URI: uri, |