1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
|
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package util
import (
"unicode"
"unicode/utf8"
"github.com/superseriousbusiness/gotosocial/internal/regexes"
)
const (
maximumHashtagLength = 30
)
// DeriveMentionNamesFromText takes a plaintext (ie., not html-formatted) text,
// and applies a regex to it to return a deduplicated list of account names
// mentioned in that text, in the format "@user@example.org" or "@username" for
// local users.
func DeriveMentionNamesFromText(text string) []string {
mentionedAccounts := []string{}
for _, m := range regexes.MentionFinder.FindAllStringSubmatch(text, -1) {
mentionedAccounts = append(mentionedAccounts, m[1])
}
return UniqueStrings(mentionedAccounts)
}
type Pair[A, B any] struct {
First A
Second B
}
// Byte index in original string
// `First` includes `#`.
type Span = Pair[int, int]
// Takes a plaintext (ie., not HTML-formatted) text,
// and returns a slice of unique hashtags.
func DeriveHashtagsFromText(text string) []string {
tagsMap := make(map[string]bool)
tags := []string{}
for _, v := range FindHashtagSpansInText(text) {
t := text[v.First+1 : v.Second]
if _, value := tagsMap[t]; !value {
tagsMap[t] = true
tags = append(tags, t)
}
}
return tags
}
// Takes a plaintext (ie., not HTML-formatted) text,
// and returns a list of pairs of indices into the original string, where
// hashtags are located.
func FindHashtagSpansInText(text string) []Span {
tags := []Span{}
start := 0
// Keep one rune of lookbehind.
prev := ' '
inTag := false
for i, r := range text {
if r == '#' && IsHashtagBoundary(prev) {
// Start of hashtag.
inTag = true
start = i
} else if inTag && !IsPermittedInHashtag(r) && !IsHashtagBoundary(r) {
// Inside the hashtag, but it was a phoney, gottem.
inTag = false
} else if inTag && IsHashtagBoundary(r) {
// End of hashtag.
inTag = false
appendTag(&tags, text, start, i)
} else if irl := i + utf8.RuneLen(r); inTag && irl == len(text) {
// End of text.
appendTag(&tags, text, start, irl)
}
prev = r
}
return tags
}
func appendTag(tags *[]Span, text string, start int, end int) {
l := end - start - 1
// This check could be moved out into the parsing loop if necessary!
if 0 < l && l <= maximumHashtagLength {
*tags = append(*tags, Span{First: start, Second: end})
}
}
// DeriveEmojisFromText takes a plaintext (ie., not html-formatted) text,
// and applies a regex to it to return a deduplicated list of emojis
// used in that text, without the surrounding `::`
func DeriveEmojisFromText(text string) []string {
emojis := []string{}
for _, m := range regexes.EmojiFinder.FindAllStringSubmatch(text, -1) {
emojis = append(emojis, m[1])
}
return UniqueStrings(emojis)
}
func IsPermittedInHashtag(r rune) bool {
return unicode.IsLetter(r) || unicode.IsNumber(r)
}
// Decides where to break before or after a hashtag.
func IsHashtagBoundary(r rune) bool {
return r == '#' || // `###lol` should work
unicode.IsSpace(r) || // All kinds of Unicode whitespace.
unicode.IsControl(r) || // All kinds of control characters, like tab.
// Most kinds of punctuation except "Pc" ("Punctuation, connecting", like `_`).
// But `someurl/#fragment` should not match, neither should HTML entities like `#`.
('/' != r && '&' != r && !unicode.Is(unicode.Categories["Pc"], r) && unicode.IsPunct(r))
}
|
