diff options
Diffstat (limited to 'internal/ap')
| -rw-r--r-- | internal/ap/ap_test.go | 91 | ||||
| -rw-r--r-- | internal/ap/extract.go | 43 | ||||
| -rw-r--r-- | internal/ap/extracthashtags_test.go | 66 |
3 files changed, 182 insertions, 18 deletions
diff --git a/internal/ap/ap_test.go b/internal/ap/ap_test.go index 105bc1fcf..6a5073c63 100644 --- a/internal/ap/ap_test.go +++ b/internal/ap/ap_test.go @@ -98,6 +98,97 @@ func noteWithMentions1() vocab.ActivityStreamsNote { return note } +func (suite *APTestSuite) noteWithHashtags1() ap.Statusable { + noteJson := []byte(` +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + { + "ostatus": "http://ostatus.org#", + "atomUri": "ostatus:atomUri", + "inReplyToAtomUri": "ostatus:inReplyToAtomUri", + "conversation": "ostatus:conversation", + "sensitive": "as:sensitive", + "toot": "http://joinmastodon.org/ns#", + "votersCount": "toot:votersCount", + "Hashtag": "as:Hashtag" + } + ], + "id": "https://mastodon.social/users/pixelfed/statuses/110609702372389319", + "type": "Note", + "summary": null, + "inReplyTo": null, + "published": "2023-06-26T09:01:56Z", + "url": "https://mastodon.social/@pixelfed/110609702372389319", + "attributedTo": "https://mastodon.social/users/pixelfed", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [ + "https://mastodon.social/users/pixelfed/followers", + "https://gts.superseriousbusiness.org/users/gotosocial" + ], + "sensitive": false, + "atomUri": "https://mastodon.social/users/pixelfed/statuses/110609702372389319", + "inReplyToAtomUri": null, + "conversation": "tag:mastodon.social,2023-06-26:objectId=474977189:objectType=Conversation", + "content": "<p>⚡ Heard of <span class=\"h-card\" translate=\"no\"><a href=\"https://gts.superseriousbusiness.org/@gotosocial\" class=\"u-url mention\">@<span>gotosocial</span></a></span> ?</p><p>GoToSocial provides a lightweight, customizable, and safety-focused entryway into the <a href=\"https://mastodon.social/tags/fediverse\" class=\"mention hashtag\" rel=\"tag\">#<span>fediverse</span></a>, you can keep in touch with your friends, post, read, and share images and articles.</p><p>Consider <a href=\"https://mastodon.social/tags/GoToSocial\" class=\"mention hashtag\" rel=\"tag\">#<span>GoToSocial</span></a> instead of Pixelfed if you'd like a safety-focused alternative with text-only post support that is maintained by a stellar developer community!</p><p>We ❤️ GtS, check them out!</p><p>🌍 <a href=\"https://gotosocial.org/\" target=\"_blank\" rel=\"nofollow noopener noreferrer\" translate=\"no\"><span class=\"invisible\">https://</span><span class=\"\">gotosocial.org/</span><span class=\"invisible\"></span></a></p><p>🔍 <a href=\"https://fedidb.org/software/gotosocial\" target=\"_blank\" rel=\"nofollow noopener noreferrer\" translate=\"no\"><span class=\"invisible\">https://</span><span class=\"\">fedidb.org/software/gotosocial</span><span class=\"invisible\"></span></a></p>", + "contentMap": { + "en": "<p>⚡ Heard of <span class=\"h-card\" translate=\"no\"><a href=\"https://gts.superseriousbusiness.org/@gotosocial\" class=\"u-url mention\">@<span>gotosocial</span></a></span> ?</p><p>GoToSocial provides a lightweight, customizable, and safety-focused entryway into the <a href=\"https://mastodon.social/tags/fediverse\" class=\"mention hashtag\" rel=\"tag\">#<span>fediverse</span></a>, you can keep in touch with your friends, post, read, and share images and articles.</p><p>Consider <a href=\"https://mastodon.social/tags/GoToSocial\" class=\"mention hashtag\" rel=\"tag\">#<span>GoToSocial</span></a> instead of Pixelfed if you'd like a safety-focused alternative with text-only post support that is maintained by a stellar developer community!</p><p>We ❤️ GtS, check them out!</p><p>🌍 <a href=\"https://gotosocial.org/\" target=\"_blank\" rel=\"nofollow noopener noreferrer\" translate=\"no\"><span class=\"invisible\">https://</span><span class=\"\">gotosocial.org/</span><span class=\"invisible\"></span></a></p><p>🔍 <a href=\"https://fedidb.org/software/gotosocial\" target=\"_blank\" rel=\"nofollow noopener noreferrer\" translate=\"no\"><span class=\"invisible\">https://</span><span class=\"\">fedidb.org/software/gotosocial</span><span class=\"invisible\"></span></a></p>" + }, + "attachment": [], + "tag": [ + { + "type": "Mention", + "href": "https://gts.superseriousbusiness.org/users/gotosocial", + "name": "@gotosocial@superseriousbusiness.org" + }, + { + "type": "Hashtag", + "href": "https://mastodon.social/tags/fediverse", + "name": "#fediverse" + }, + { + "type": "Hashtag", + "href": "https://mastodon.social/tags/gotosocial", + "name": "#gotosocial" + }, + { + "type": "Hashtag", + "href": "https://mastodon.social/tags/this_hashtag_will_be_ignored_since_it_cant_be_normalized", + "name": "#b̴̛͇̒̌͑̓̐̑͗̏̐̇͗̎̕͝O̵̧̧͎̟̰̭̊͌͒́̊̑̄̐͐͗Ọ̷̧̡̰̟̪̫̹͖͇̱͕̺̦̲̀̐̽̓̇̚͠b̶̨̖͍͙͈̹͉̯͕̯̯̯̞̼̞̏͊͂̐̔͛s̴̢̞̺͈͇̘͚͉͔̥̔͛͆͑͑̍̄̌̚͜͜ͅ" + }, + { + "type": "Hashtag", + "href": "https://mastodon.social/tags/this_hashtag_will_be_included_correctly", + "name": "#Grüvy" + }, + { + "type": "Hashtag", + "href": "https://mastodon.social/tags/this_hashtag_will_be_squashed_into_a_single_character", + "name": "#` + `ᄀ` + `ᅡ` + `ᆨ` + `" + } + ], + "replies": { + "id": "https://mastodon.social/users/pixelfed/statuses/110609702372389319/replies", + "type": "Collection", + "first": { + "type": "CollectionPage", + "next": "https://mastodon.social/users/pixelfed/statuses/110609702372389319/replies?only_other_accounts=true&page=true", + "partOf": "https://mastodon.social/users/pixelfed/statuses/110609702372389319/replies", + "items": [] + } + } +}`) + + statusable, err := ap.ResolveStatusable(context.Background(), noteJson) + if err != nil { + suite.FailNow(err.Error()) + } + + return statusable +} + func addressable1() ap.Addressable { // make a note addressed to public with followers in cc note := streams.NewActivityStreamsNote() diff --git a/internal/ap/extract.go b/internal/ap/extract.go index 9a1b6aa4f..21ff20235 100644 --- a/internal/ap/extract.go +++ b/internal/ap/extract.go @@ -30,6 +30,7 @@ import ( "github.com/superseriousbusiness/activity/pub" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/text" "github.com/superseriousbusiness/gotosocial/internal/util" ) @@ -529,7 +530,7 @@ func ExtractBlurhash(i WithBlurhash) string { // ExtractHashtags extracts a slice of minimal gtsmodel.Tags // from a WithTag. If an entry in the WithTag is not a hashtag, -// it will be quietly ignored. +// or has a name that cannot be normalized, it will be ignored. // // TODO: find a better heuristic for determining if something // is a hashtag or not, since looking for type name "Hashtag" @@ -562,18 +563,29 @@ func ExtractHashtags(i WithTag) ([]*gtsmodel.Tag, error) { continue } - tag, err := ExtractHashtag(hashtaggable) + tag, err := extractHashtag(hashtaggable) if err != nil { continue } + // "Normalize" this tag by combining diacritics + + // unicode chars. If this returns false, it means + // we couldn't normalize it well enough to make it + // valid on our instance, so just ignore it. + normalized, ok := text.NormalizeHashtag(tag.Name) + if !ok { + continue + } + + // We store tag names lowercased, might + // as well change case here already. + tag.Name = strings.ToLower(normalized) + // Only append this tag if we haven't // seen it already, to avoid duplicates // in the slice. - if _, set := keys[tag.URL]; !set { - keys[tag.URL] = nil // Value doesn't matter. - tags = append(tags, tag) - tags = append(tags, tag) + if _, set := keys[tag.Name]; !set { + keys[tag.Name] = nil // Value doesn't matter. tags = append(tags, tag) } } @@ -581,16 +593,9 @@ func ExtractHashtags(i WithTag) ([]*gtsmodel.Tag, error) { return tags, nil } -// ExtractEmoji extracts a minimal gtsmodel.Tag -// from the given Hashtaggable. -func ExtractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) { - // Extract href/link for this tag. - hrefProp := i.GetActivityStreamsHref() - if hrefProp == nil || !hrefProp.IsIRI() { - return nil, gtserror.New("no href prop") - } - tagURL := hrefProp.GetIRI().String() - +// extractHashtag extracts a minimal gtsmodel.Tag from the given +// Hashtaggable, without yet doing any normalization on it. +func extractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) { // Extract name for the tag; trim leading hash // character, so '#example' becomes 'example'. name := ExtractName(i) @@ -599,9 +604,11 @@ func ExtractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) { } tagName := strings.TrimPrefix(name, "#") + yeah := func() *bool { t := true; return &t } return >smodel.Tag{ - URL: tagURL, - Name: tagName, + Name: tagName, + Useable: yeah(), // Assume true by default. + Listable: yeah(), // Assume true by default. }, nil } diff --git a/internal/ap/extracthashtags_test.go b/internal/ap/extracthashtags_test.go new file mode 100644 index 000000000..1d4fbcf6f --- /dev/null +++ b/internal/ap/extracthashtags_test.go @@ -0,0 +1,66 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package ap_test + +import ( + "testing" + + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/ap" +) + +type ExtractHashtagsTestSuite struct { + APTestSuite +} + +func (suite *ExtractHashtagsTestSuite) TestExtractHashtags1() { + note := suite.noteWithHashtags1() + + hashtags, err := ap.ExtractHashtags(note) + if err != nil { + suite.FailNow(err.Error()) + } + + if l := len(hashtags); l != 4 { + suite.FailNow("", "expected 4 hashtags, got %d", l) + } + + hashtagFediverse := hashtags[0] + suite.Equal("fediverse", hashtagFediverse.Name) + suite.Equal(true, *hashtagFediverse.Useable) + suite.Equal(true, *hashtagFediverse.Listable) + + hashtagGoToSocial := hashtags[1] + suite.Equal("gotosocial", hashtagGoToSocial.Name) + suite.Equal(true, *hashtagGoToSocial.Useable) + suite.Equal(true, *hashtagGoToSocial.Listable) + + hashtagGrüvy := hashtags[2] + suite.Equal("grüvy", hashtagGrüvy.Name) + suite.Equal(true, *hashtagGrüvy.Useable) + suite.Equal(true, *hashtagGrüvy.Listable) + + hashtagAngle := hashtags[3] + suite.Equal("각", hashtagAngle.Name) + suite.Equal(true, *hashtagAngle.Useable) + suite.Equal(true, *hashtagAngle.Listable) +} + +func TestExtractHashtagsTestSuite(t *testing.T) { + suite.Run(t, &ExtractHashtagsTestSuite{}) +} |