summaryrefslogtreecommitdiff
path: root/internal/ap
diff options
context:
space:
mode:
authorLibravatar tobi <31960611+tsmethurst@users.noreply.github.com>2023-07-31 15:47:35 +0200
committerLibravatar GitHub <noreply@github.com>2023-07-31 15:47:35 +0200
commit2796a2e82f16ade9872008878cf88299bd66b4e7 (patch)
tree76f7b69cc1da57ca10b71c57abf1892575bea100 /internal/ap
parent[performance] cache follow, follow request and block ID lists (#2027) (diff)
downloadgotosocial-2796a2e82f16ade9872008878cf88299bd66b4e7.tar.xz
[feature] Hashtag federation (in/out), hashtag client API endpoints (#2032)
* update go-fed * do the things * remove unused columns from tags * update to latest lingo from main * further tag shenanigans * serve stub page at tag endpoint * we did it lads * tests, oh tests, ohhh tests, oh tests (doo doo doo doo) * swagger docs * document hashtag usage + federation * instanceGet * don't bother parsing tag href * rename whereStartsWith -> whereStartsLike * remove GetOrCreateTag * dont cache status tag timelineability
Diffstat (limited to 'internal/ap')
-rw-r--r--internal/ap/ap_test.go91
-rw-r--r--internal/ap/extract.go43
-rw-r--r--internal/ap/extracthashtags_test.go66
3 files changed, 182 insertions, 18 deletions
diff --git a/internal/ap/ap_test.go b/internal/ap/ap_test.go
index 105bc1fcf..6a5073c63 100644
--- a/internal/ap/ap_test.go
+++ b/internal/ap/ap_test.go
@@ -98,6 +98,97 @@ func noteWithMentions1() vocab.ActivityStreamsNote {
return note
}
+func (suite *APTestSuite) noteWithHashtags1() ap.Statusable {
+ noteJson := []byte(`
+{
+ "@context": [
+ "https://www.w3.org/ns/activitystreams",
+ {
+ "ostatus": "http://ostatus.org#",
+ "atomUri": "ostatus:atomUri",
+ "inReplyToAtomUri": "ostatus:inReplyToAtomUri",
+ "conversation": "ostatus:conversation",
+ "sensitive": "as:sensitive",
+ "toot": "http://joinmastodon.org/ns#",
+ "votersCount": "toot:votersCount",
+ "Hashtag": "as:Hashtag"
+ }
+ ],
+ "id": "https://mastodon.social/users/pixelfed/statuses/110609702372389319",
+ "type": "Note",
+ "summary": null,
+ "inReplyTo": null,
+ "published": "2023-06-26T09:01:56Z",
+ "url": "https://mastodon.social/@pixelfed/110609702372389319",
+ "attributedTo": "https://mastodon.social/users/pixelfed",
+ "to": [
+ "https://www.w3.org/ns/activitystreams#Public"
+ ],
+ "cc": [
+ "https://mastodon.social/users/pixelfed/followers",
+ "https://gts.superseriousbusiness.org/users/gotosocial"
+ ],
+ "sensitive": false,
+ "atomUri": "https://mastodon.social/users/pixelfed/statuses/110609702372389319",
+ "inReplyToAtomUri": null,
+ "conversation": "tag:mastodon.social,2023-06-26:objectId=474977189:objectType=Conversation",
+ "content": "<p>⚡ Heard of <span class=\"h-card\" translate=\"no\"><a href=\"https://gts.superseriousbusiness.org/@gotosocial\" class=\"u-url mention\">@<span>gotosocial</span></a></span> ?</p><p>GoToSocial provides a lightweight, customizable, and safety-focused entryway into the <a href=\"https://mastodon.social/tags/fediverse\" class=\"mention hashtag\" rel=\"tag\">#<span>fediverse</span></a>, you can keep in touch with your friends, post, read, and share images and articles.</p><p>Consider <a href=\"https://mastodon.social/tags/GoToSocial\" class=\"mention hashtag\" rel=\"tag\">#<span>GoToSocial</span></a> instead of Pixelfed if you&#39;d like a safety-focused alternative with text-only post support that is maintained by a stellar developer community!</p><p>We ❤️ GtS, check them out!</p><p>🌍 <a href=\"https://gotosocial.org/\" target=\"_blank\" rel=\"nofollow noopener noreferrer\" translate=\"no\"><span class=\"invisible\">https://</span><span class=\"\">gotosocial.org/</span><span class=\"invisible\"></span></a></p><p>🔍 <a href=\"https://fedidb.org/software/gotosocial\" target=\"_blank\" rel=\"nofollow noopener noreferrer\" translate=\"no\"><span class=\"invisible\">https://</span><span class=\"\">fedidb.org/software/gotosocial</span><span class=\"invisible\"></span></a></p>",
+ "contentMap": {
+ "en": "<p>⚡ Heard of <span class=\"h-card\" translate=\"no\"><a href=\"https://gts.superseriousbusiness.org/@gotosocial\" class=\"u-url mention\">@<span>gotosocial</span></a></span> ?</p><p>GoToSocial provides a lightweight, customizable, and safety-focused entryway into the <a href=\"https://mastodon.social/tags/fediverse\" class=\"mention hashtag\" rel=\"tag\">#<span>fediverse</span></a>, you can keep in touch with your friends, post, read, and share images and articles.</p><p>Consider <a href=\"https://mastodon.social/tags/GoToSocial\" class=\"mention hashtag\" rel=\"tag\">#<span>GoToSocial</span></a> instead of Pixelfed if you&#39;d like a safety-focused alternative with text-only post support that is maintained by a stellar developer community!</p><p>We ❤️ GtS, check them out!</p><p>🌍 <a href=\"https://gotosocial.org/\" target=\"_blank\" rel=\"nofollow noopener noreferrer\" translate=\"no\"><span class=\"invisible\">https://</span><span class=\"\">gotosocial.org/</span><span class=\"invisible\"></span></a></p><p>🔍 <a href=\"https://fedidb.org/software/gotosocial\" target=\"_blank\" rel=\"nofollow noopener noreferrer\" translate=\"no\"><span class=\"invisible\">https://</span><span class=\"\">fedidb.org/software/gotosocial</span><span class=\"invisible\"></span></a></p>"
+ },
+ "attachment": [],
+ "tag": [
+ {
+ "type": "Mention",
+ "href": "https://gts.superseriousbusiness.org/users/gotosocial",
+ "name": "@gotosocial@superseriousbusiness.org"
+ },
+ {
+ "type": "Hashtag",
+ "href": "https://mastodon.social/tags/fediverse",
+ "name": "#fediverse"
+ },
+ {
+ "type": "Hashtag",
+ "href": "https://mastodon.social/tags/gotosocial",
+ "name": "#gotosocial"
+ },
+ {
+ "type": "Hashtag",
+ "href": "https://mastodon.social/tags/this_hashtag_will_be_ignored_since_it_cant_be_normalized",
+ "name": "#b̴̛͇̒̌͑̓̐̑͗̏̐̇͗̎̕͝O̵̧̧͎̟̰̭̊͌͒́̊̑̄̐͐͗Ọ̷̧̡̰̟̪̫̹͖͇̱͕̺̦̲̀̐̽̓̇̚͠b̶̨̖͍͙͈̹͉̯͕̯̯̯̞̼̞̏͊͂̐̔͛s̴̢̞̺͈͇̘͚͉͔̥̔͛͆͑͑̍̄̌̚͜͜ͅ"
+ },
+ {
+ "type": "Hashtag",
+ "href": "https://mastodon.social/tags/this_hashtag_will_be_included_correctly",
+ "name": "#Grüvy"
+ },
+ {
+ "type": "Hashtag",
+ "href": "https://mastodon.social/tags/this_hashtag_will_be_squashed_into_a_single_character",
+ "name": "#` + `ᄀ` + `ᅡ` + `ᆨ` + `"
+ }
+ ],
+ "replies": {
+ "id": "https://mastodon.social/users/pixelfed/statuses/110609702372389319/replies",
+ "type": "Collection",
+ "first": {
+ "type": "CollectionPage",
+ "next": "https://mastodon.social/users/pixelfed/statuses/110609702372389319/replies?only_other_accounts=true&page=true",
+ "partOf": "https://mastodon.social/users/pixelfed/statuses/110609702372389319/replies",
+ "items": []
+ }
+ }
+}`)
+
+ statusable, err := ap.ResolveStatusable(context.Background(), noteJson)
+ if err != nil {
+ suite.FailNow(err.Error())
+ }
+
+ return statusable
+}
+
func addressable1() ap.Addressable {
// make a note addressed to public with followers in cc
note := streams.NewActivityStreamsNote()
diff --git a/internal/ap/extract.go b/internal/ap/extract.go
index 9a1b6aa4f..21ff20235 100644
--- a/internal/ap/extract.go
+++ b/internal/ap/extract.go
@@ -30,6 +30,7 @@ import (
"github.com/superseriousbusiness/activity/pub"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+ "github.com/superseriousbusiness/gotosocial/internal/text"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
@@ -529,7 +530,7 @@ func ExtractBlurhash(i WithBlurhash) string {
// ExtractHashtags extracts a slice of minimal gtsmodel.Tags
// from a WithTag. If an entry in the WithTag is not a hashtag,
-// it will be quietly ignored.
+// or has a name that cannot be normalized, it will be ignored.
//
// TODO: find a better heuristic for determining if something
// is a hashtag or not, since looking for type name "Hashtag"
@@ -562,18 +563,29 @@ func ExtractHashtags(i WithTag) ([]*gtsmodel.Tag, error) {
continue
}
- tag, err := ExtractHashtag(hashtaggable)
+ tag, err := extractHashtag(hashtaggable)
if err != nil {
continue
}
+ // "Normalize" this tag by combining diacritics +
+ // unicode chars. If this returns false, it means
+ // we couldn't normalize it well enough to make it
+ // valid on our instance, so just ignore it.
+ normalized, ok := text.NormalizeHashtag(tag.Name)
+ if !ok {
+ continue
+ }
+
+ // We store tag names lowercased, might
+ // as well change case here already.
+ tag.Name = strings.ToLower(normalized)
+
// Only append this tag if we haven't
// seen it already, to avoid duplicates
// in the slice.
- if _, set := keys[tag.URL]; !set {
- keys[tag.URL] = nil // Value doesn't matter.
- tags = append(tags, tag)
- tags = append(tags, tag)
+ if _, set := keys[tag.Name]; !set {
+ keys[tag.Name] = nil // Value doesn't matter.
tags = append(tags, tag)
}
}
@@ -581,16 +593,9 @@ func ExtractHashtags(i WithTag) ([]*gtsmodel.Tag, error) {
return tags, nil
}
-// ExtractEmoji extracts a minimal gtsmodel.Tag
-// from the given Hashtaggable.
-func ExtractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) {
- // Extract href/link for this tag.
- hrefProp := i.GetActivityStreamsHref()
- if hrefProp == nil || !hrefProp.IsIRI() {
- return nil, gtserror.New("no href prop")
- }
- tagURL := hrefProp.GetIRI().String()
-
+// extractHashtag extracts a minimal gtsmodel.Tag from the given
+// Hashtaggable, without yet doing any normalization on it.
+func extractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) {
// Extract name for the tag; trim leading hash
// character, so '#example' becomes 'example'.
name := ExtractName(i)
@@ -599,9 +604,11 @@ func ExtractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) {
}
tagName := strings.TrimPrefix(name, "#")
+ yeah := func() *bool { t := true; return &t }
return &gtsmodel.Tag{
- URL: tagURL,
- Name: tagName,
+ Name: tagName,
+ Useable: yeah(), // Assume true by default.
+ Listable: yeah(), // Assume true by default.
}, nil
}
diff --git a/internal/ap/extracthashtags_test.go b/internal/ap/extracthashtags_test.go
new file mode 100644
index 000000000..1d4fbcf6f
--- /dev/null
+++ b/internal/ap/extracthashtags_test.go
@@ -0,0 +1,66 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package ap_test
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/suite"
+ "github.com/superseriousbusiness/gotosocial/internal/ap"
+)
+
+type ExtractHashtagsTestSuite struct {
+ APTestSuite
+}
+
+func (suite *ExtractHashtagsTestSuite) TestExtractHashtags1() {
+ note := suite.noteWithHashtags1()
+
+ hashtags, err := ap.ExtractHashtags(note)
+ if err != nil {
+ suite.FailNow(err.Error())
+ }
+
+ if l := len(hashtags); l != 4 {
+ suite.FailNow("", "expected 4 hashtags, got %d", l)
+ }
+
+ hashtagFediverse := hashtags[0]
+ suite.Equal("fediverse", hashtagFediverse.Name)
+ suite.Equal(true, *hashtagFediverse.Useable)
+ suite.Equal(true, *hashtagFediverse.Listable)
+
+ hashtagGoToSocial := hashtags[1]
+ suite.Equal("gotosocial", hashtagGoToSocial.Name)
+ suite.Equal(true, *hashtagGoToSocial.Useable)
+ suite.Equal(true, *hashtagGoToSocial.Listable)
+
+ hashtagGrüvy := hashtags[2]
+ suite.Equal("grüvy", hashtagGrüvy.Name)
+ suite.Equal(true, *hashtagGrüvy.Useable)
+ suite.Equal(true, *hashtagGrüvy.Listable)
+
+ hashtagAngle := hashtags[3]
+ suite.Equal("각", hashtagAngle.Name)
+ suite.Equal(true, *hashtagAngle.Useable)
+ suite.Equal(true, *hashtagAngle.Listable)
+}
+
+func TestExtractHashtagsTestSuite(t *testing.T) {
+ suite.Run(t, &ExtractHashtagsTestSuite{})
+}