1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
|
/*
GoToSocial
Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package media
import (
"context"
"errors"
"fmt"
"io"
"net/url"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
type DereferenceMedia func(ctx context.Context, iri *url.URL) (io.ReadCloser, int64, error)
func (m *manager) RefetchEmojis(ctx context.Context, domain string, dereferenceMedia DereferenceMedia) (int, error) {
// normalize domain
if domain == "" {
domain = db.EmojiAllDomains
}
var (
maxShortcodeDomain string
refetchIDs []string
)
// page through emojis 20 at a time, looking for those with missing images
for {
// Fetch next block of emojis from database
emojis, err := m.state.DB.GetEmojis(ctx, domain, false, true, "", maxShortcodeDomain, "", 20)
if err != nil {
if !errors.Is(err, db.ErrNoEntries) {
// an actual error has occurred
log.Errorf("error fetching emojis from database: %s", err)
}
break
}
for _, emoji := range emojis {
if emoji.Domain == "" {
// never try to refetch local emojis
continue
}
if refetch, err := m.emojiRequiresRefetch(ctx, emoji); err != nil {
// an error here indicates something is wrong with storage, so we should stop
return 0, fmt.Errorf("error checking refetch requirement for emoji %s: %w", util.ShortcodeDomain(emoji), err)
} else if !refetch {
continue
}
refetchIDs = append(refetchIDs, emoji.ID)
}
// Update next maxShortcodeDomain from last emoji
maxShortcodeDomain = util.ShortcodeDomain(emojis[len(emojis)-1])
}
// bail early if we've got nothing to do
toRefetchCount := len(refetchIDs)
if toRefetchCount == 0 {
log.Debug("no remote emojis require a refetch")
return 0, nil
}
log.Debugf("%d remote emoji(s) require a refetch, doing that now...", toRefetchCount)
var totalRefetched int
for _, emojiID := range refetchIDs {
emoji, err := m.state.DB.GetEmojiByID(ctx, emojiID)
if err != nil {
// this shouldn't happen--since we know we have the emoji--so return if it does
return 0, fmt.Errorf("error getting emoji %s: %w", emojiID, err)
}
shortcodeDomain := util.ShortcodeDomain(emoji)
if emoji.ImageRemoteURL == "" {
log.Errorf("remote emoji %s could not be refreshed because it has no ImageRemoteURL set", shortcodeDomain)
continue
}
emojiImageIRI, err := url.Parse(emoji.ImageRemoteURL)
if err != nil {
log.Errorf("remote emoji %s could not be refreshed because its ImageRemoteURL (%s) is not a valid uri: %s", shortcodeDomain, emoji.ImageRemoteURL, err)
continue
}
dataFunc := func(ctx context.Context) (reader io.ReadCloser, fileSize int64, err error) {
return dereferenceMedia(ctx, emojiImageIRI)
}
processingEmoji, err := m.PreProcessEmoji(ctx, dataFunc, nil, emoji.Shortcode, emoji.ID, emoji.URI, &AdditionalEmojiInfo{
Domain: &emoji.Domain,
ImageRemoteURL: &emoji.ImageRemoteURL,
ImageStaticRemoteURL: &emoji.ImageStaticRemoteURL,
Disabled: emoji.Disabled,
VisibleInPicker: emoji.VisibleInPicker,
}, true)
if err != nil {
log.Errorf("emoji %s could not be refreshed because of an error during processing: %s", shortcodeDomain, err)
continue
}
if _, err := processingEmoji.LoadEmoji(ctx); err != nil {
log.Errorf("emoji %s could not be refreshed because of an error during loading: %s", shortcodeDomain, err)
continue
}
log.Tracef("refetched emoji %s successfully from remote", shortcodeDomain)
totalRefetched++
}
return totalRefetched, nil
}
func (m *manager) emojiRequiresRefetch(ctx context.Context, emoji *gtsmodel.Emoji) (bool, error) {
if has, err := m.state.Storage.Has(ctx, emoji.ImagePath); err != nil {
return false, err
} else if !has {
return true, nil
}
if has, err := m.state.Storage.Has(ctx, emoji.ImageStaticPath); err != nil {
return false, err
} else if !has {
return true, nil
}
return false, nil
}
|
