summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--cmd/gotosocial/action/admin/media/list.go168
-rw-r--r--cmd/gotosocial/admin.go22
-rw-r--r--docs/admin/cli.md68
-rw-r--r--internal/config/config.go12
-rw-r--r--internal/config/flags.go11
-rw-r--r--internal/config/helpers.gen.go50
-rwxr-xr-xtest/envparsing.sh2
7 files changed, 282 insertions, 51 deletions
diff --git a/cmd/gotosocial/action/admin/media/list.go b/cmd/gotosocial/action/admin/media/list.go
index 5b6108b11..0a2e60ede 100644
--- a/cmd/gotosocial/action/admin/media/list.go
+++ b/cmd/gotosocial/action/admin/media/list.go
@@ -20,6 +20,7 @@ package media
import (
"bufio"
"context"
+ "errors"
"fmt"
"os"
"path"
@@ -34,14 +35,17 @@ import (
)
type list struct {
- dbService db.DB
- state *state.State
- maxID string
- limit int
- out *bufio.Writer
+ dbService db.DB
+ state *state.State
+ maxID string
+ limit int
+ localOnly bool
+ remoteOnly bool
+ out *bufio.Writer
}
-func (l *list) GetAllMediaPaths(ctx context.Context, filter func(*gtsmodel.MediaAttachment) string) ([]string, error) {
+// Get a list of attachment using a custom filter
+func (l *list) GetAllAttachmentPaths(ctx context.Context, filter func(*gtsmodel.MediaAttachment) string) ([]string, error) {
res := make([]string, 0, 100)
for {
attachments, err := l.dbService.GetAttachments(ctx, l.maxID, l.limit)
@@ -72,8 +76,52 @@ func (l *list) GetAllMediaPaths(ctx context.Context, filter func(*gtsmodel.Media
return res, nil
}
+// Get a list of emojis using a custom filter
+func (l *list) GetAllEmojisPaths(ctx context.Context, filter func(*gtsmodel.Emoji) string) ([]string, error) {
+ res := make([]string, 0, 100)
+ for {
+ attachments, err := l.dbService.GetEmojis(ctx, l.maxID, l.limit)
+ if err != nil {
+ return nil, fmt.Errorf("failed to retrieve media metadata from database: %w", err)
+ }
+
+ for _, a := range attachments {
+ v := filter(a)
+ if v != "" {
+ res = append(res, v)
+ }
+ }
+
+ // If we got less results than our limit, we've reached the
+ // last page to retrieve and we can break the loop. If the
+ // last batch happens to contain exactly the same amount of
+ // items as the limit we'll end up doing one extra query.
+ if len(attachments) < l.limit {
+ break
+ }
+
+ // Grab the last ID from the batch and set it as the maxID
+ // that'll be used in the next iteration so we don't get items
+ // we've already seen.
+ l.maxID = attachments[len(attachments)-1].ID
+ }
+ return res, nil
+}
+
func setupList(ctx context.Context) (*list, error) {
- var state state.State
+ var (
+ localOnly = config.GetAdminMediaListLocalOnly()
+ remoteOnly = config.GetAdminMediaListRemoteOnly()
+ state state.State
+ )
+
+ // Validate flags.
+ if localOnly && remoteOnly {
+ return nil, errors.New(
+ "local-only and remote-only flags cannot be true at the same time; " +
+ "choose one or the other, or set neither to list all media",
+ )
+ }
state.Caches.Init()
state.Caches.Start()
@@ -87,11 +135,13 @@ func setupList(ctx context.Context) (*list, error) {
state.DB = dbService
return &list{
- dbService: dbService,
- state: &state,
- limit: 200,
- maxID: "",
- out: bufio.NewWriter(os.Stdout),
+ dbService: dbService,
+ state: &state,
+ limit: 200,
+ maxID: "",
+ localOnly: localOnly,
+ remoteOnly: remoteOnly,
+ out: bufio.NewWriter(os.Stdout),
}, nil
}
@@ -103,7 +153,8 @@ func (l *list) shutdown() error {
return err
}
-var ListLocal action.GTSAction = func(ctx context.Context) error {
+// ListAttachments lists local, remote, or all attachment paths.
+var ListAttachments action.GTSAction = func(ctx context.Context) error {
list, err := setupList(ctx)
if err != nil {
return err
@@ -116,26 +167,53 @@ var ListLocal action.GTSAction = func(ctx context.Context) error {
}
}()
- mediaPath := config.GetStorageLocalBasePath()
- media, err := list.GetAllMediaPaths(
- ctx,
- func(m *gtsmodel.MediaAttachment) string {
+ var (
+ mediaPath = config.GetStorageLocalBasePath()
+ filter func(*gtsmodel.MediaAttachment) string
+ )
+
+ switch {
+ case list.localOnly:
+ filter = func(m *gtsmodel.MediaAttachment) string {
+ if m.RemoteURL != "" {
+ // Remote, not
+ // interested.
+ return ""
+ }
+
+ return path.Join(mediaPath, m.File.Path)
+ }
+
+ case list.remoteOnly:
+ filter = func(m *gtsmodel.MediaAttachment) string {
if m.RemoteURL == "" {
- return path.Join(mediaPath, m.File.Path)
+ // Local, not
+ // interested.
+ return ""
}
- return ""
- })
+
+ return path.Join(mediaPath, m.File.Path)
+ }
+
+ default:
+ filter = func(m *gtsmodel.MediaAttachment) string {
+ return path.Join(mediaPath, m.File.Path)
+ }
+ }
+
+ attachments, err := list.GetAllAttachmentPaths(ctx, filter)
if err != nil {
return err
}
- for _, m := range media {
- _, _ = list.out.WriteString(m + "\n")
+ for _, a := range attachments {
+ _, _ = list.out.WriteString(a + "\n")
}
return nil
}
-var ListRemote action.GTSAction = func(ctx context.Context) error {
+// ListEmojis lists local, remote, or all emoji filepaths.
+var ListEmojis action.GTSAction = func(ctx context.Context) error {
list, err := setupList(ctx)
if err != nil {
return err
@@ -148,17 +226,47 @@ var ListRemote action.GTSAction = func(ctx context.Context) error {
}
}()
- media, err := list.GetAllMediaPaths(
- ctx,
- func(m *gtsmodel.MediaAttachment) string {
- return m.RemoteURL
- })
+ var (
+ mediaPath = config.GetStorageLocalBasePath()
+ filter func(*gtsmodel.Emoji) string
+ )
+
+ switch {
+ case list.localOnly:
+ filter = func(e *gtsmodel.Emoji) string {
+ if e.ImageRemoteURL != "" {
+ // Remote, not
+ // interested.
+ return ""
+ }
+
+ return path.Join(mediaPath, e.ImagePath)
+ }
+
+ case list.remoteOnly:
+ filter = func(e *gtsmodel.Emoji) string {
+ if e.ImageRemoteURL == "" {
+ // Local, not
+ // interested.
+ return ""
+ }
+
+ return path.Join(mediaPath, e.ImagePath)
+ }
+
+ default:
+ filter = func(e *gtsmodel.Emoji) string {
+ return path.Join(mediaPath, e.ImagePath)
+ }
+ }
+
+ emojis, err := list.GetAllEmojisPaths(ctx, filter)
if err != nil {
return err
}
- for _, m := range media {
- _, _ = list.out.WriteString(m + "\n")
+ for _, e := range emojis {
+ _, _ = list.out.WriteString(e + "\n")
}
return nil
}
diff --git a/cmd/gotosocial/admin.go b/cmd/gotosocial/admin.go
index a8aab810e..3b2ea69e3 100644
--- a/cmd/gotosocial/admin.go
+++ b/cmd/gotosocial/admin.go
@@ -178,29 +178,31 @@ func adminCommands() *cobra.Command {
ADMIN MEDIA LIST COMMANDS
*/
- adminMediaListLocalCmd := &cobra.Command{
- Use: "list-local",
- Short: "admin command to list media on local storage",
+ adminMediaListAttachmentsCmd := &cobra.Command{
+ Use: "list-attachments",
+ Short: "list local, remote, or all attachments",
PreRunE: func(cmd *cobra.Command, args []string) error {
return preRun(preRunArgs{cmd: cmd})
},
RunE: func(cmd *cobra.Command, args []string) error {
- return run(cmd.Context(), media.ListLocal)
+ return run(cmd.Context(), media.ListAttachments)
},
}
+ config.AddAdminMediaList(adminMediaListAttachmentsCmd)
+ adminMediaCmd.AddCommand(adminMediaListAttachmentsCmd)
- adminMediaListRemoteCmd := &cobra.Command{
- Use: "list-remote",
- Short: "admin command to list remote media cached on this instance",
+ adminMediaListEmojisLocalCmd := &cobra.Command{
+ Use: "list-emojis",
+ Short: "list local, remote, or all emojis",
PreRunE: func(cmd *cobra.Command, args []string) error {
return preRun(preRunArgs{cmd: cmd})
},
RunE: func(cmd *cobra.Command, args []string) error {
- return run(cmd.Context(), media.ListRemote)
+ return run(cmd.Context(), media.ListEmojis)
},
}
-
- adminMediaCmd.AddCommand(adminMediaListLocalCmd, adminMediaListRemoteCmd)
+ config.AddAdminMediaList(adminMediaListEmojisLocalCmd)
+ adminMediaCmd.AddCommand(adminMediaListEmojisLocalCmd)
/*
ADMIN MEDIA PRUNE COMMANDS
diff --git a/docs/admin/cli.md b/docs/admin/cli.md
index 653c8aa3e..405093531 100644
--- a/docs/admin/cli.md
+++ b/docs/admin/cli.md
@@ -255,17 +255,73 @@ Example:
gotosocial admin import --path example.json --config-path config.yaml
```
-### gotosocial admin media list-local
+### gotosocial admin media list-attachments
-This command can be used to list local media. Local media is media that belongs to posts by users with an account on the instance.
+Can be used to list the storage paths of local, remote, or all media attachments on your instance (including headers and avatars).
-The output will be a list of files. The list can be used to drive your backups.
+`local-only` and `remote-only` can be used as filters; they cannot both be set at once.
-### gotosocial admin media list-remote
+If neither `local-only` or `remote-only` are set, all media attachments on your instance will be listed.
-This is the corollary to list-local, but instead lists media from remote instances. Remote media belongs to other instances, but was attached to a post we received over federation and have potentially cached locally.
+You may want to run this with `GTS_LOG_LEVEL` set to `warn` or `error`, otherwise it will log a lot of info messages you probably don't need.
-The output will be a list of URLs to retrieve the original content from. GoToSocial automatically retrieves remote media when it needs it, so you should never need to do so yourself.
+`gotosocial admin media list-attachments --help`:
+
+```text
+list local, remote, or all attachments
+
+Usage:
+ gotosocial admin media list-attachments [flags]
+
+Flags:
+ -h, --help help for list-attachments
+ --local-only list only local attachments/emojis; if specified then remote-only cannot also be true
+ --remote-only list only remote attachments/emojis; if specified then local-only cannot also be true
+```
+
+Example output:
+
+```text
+/gotosocial/062G5WYKY35KKD12EMSM3F8PJ8/attachment/original/01PFPMWK2FF0D9WMHEJHR07C3R.jpg
+/gotosocial/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg
+/gotosocial/01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpg
+/gotosocial/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpg
+/gotosocial/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH7TDVANYKWVE8VVKFPJTJ.gif
+/gotosocial/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg
+/gotosocial/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH58A357CV5K7R7TJMSH6S.jpg
+/gotosocial/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01CDR64G398ADCHXK08WWTHEZ5.gif
+```
+
+### gotosocial admin media list-emojis
+
+Can be used to list the storage paths of local, remote, or all emojis on your instance.
+
+`local-only` and `remote-only` can be used as filters; they cannot both be set at once.
+
+If neither `local-only` or `remote-only` are set, all emojis on your instance will be listed.
+
+You may want to run this with `GTS_LOG_LEVEL` set to `warn` or `error`, otherwise it will log a lot of info messages you probably don't need.
+
+`gotosocial admin media list-emojis --help`:
+
+```text
+list local, remote, or all emojis
+
+Usage:
+ gotosocial admin media list-emojis [flags]
+
+Flags:
+ -h, --help help for list-emojis
+ --local-only list only local attachments/emojis; if specified then remote-only cannot also be true
+ --remote-only list only remote attachments/emojis; if specified then local-only cannot also be true
+```
+
+Example output:
+
+```text
+/gotosocial/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01GD5KP5CQEE1R3X43Y1EHS2CW.png
+/gotosocial/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png
+```
### gotosocial admin media prune orphaned
diff --git a/internal/config/config.go b/internal/config/config.go
index 612947c5e..5da226237 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -161,11 +161,13 @@ type Configuration struct {
Cache CacheConfiguration `name:"cache"`
// TODO: move these elsewhere, these are more ephemeral vs long-running flags like above
- AdminAccountUsername string `name:"username" usage:"the username to create/delete/etc"`
- AdminAccountEmail string `name:"email" usage:"the email address of this account"`
- AdminAccountPassword string `name:"password" usage:"the password to set for this account"`
- AdminTransPath string `name:"path" usage:"the path of the file to import from/export to"`
- AdminMediaPruneDryRun bool `name:"dry-run" usage:"perform a dry run and only log number of items eligible for pruning"`
+ AdminAccountUsername string `name:"username" usage:"the username to create/delete/etc"`
+ AdminAccountEmail string `name:"email" usage:"the email address of this account"`
+ AdminAccountPassword string `name:"password" usage:"the password to set for this account"`
+ AdminTransPath string `name:"path" usage:"the path of the file to import from/export to"`
+ AdminMediaPruneDryRun bool `name:"dry-run" usage:"perform a dry run and only log number of items eligible for pruning"`
+ AdminMediaListLocalOnly bool `name:"local-only" usage:"list only local attachments/emojis; if specified then remote-only cannot also be true"`
+ AdminMediaListRemoteOnly bool `name:"remote-only" usage:"list only remote attachments/emojis; if specified then local-only cannot also be true"`
RequestIDHeader string `name:"request-id-header" usage:"Header to extract the Request ID from. Eg.,'X-Request-Id'."`
}
diff --git a/internal/config/flags.go b/internal/config/flags.go
index ad07ec2ef..74ceedc00 100644
--- a/internal/config/flags.go
+++ b/internal/config/flags.go
@@ -203,6 +203,17 @@ func AddAdminTrans(cmd *cobra.Command) {
}
}
+// AddAdminMediaList attaches flags pertaining to media list commands.
+func AddAdminMediaList(cmd *cobra.Command) {
+ localOnly := AdminMediaListLocalOnlyFlag()
+ localOnlyUsage := fieldtag("AdminMediaListLocalOnly", "usage")
+ cmd.Flags().Bool(localOnly, false, localOnlyUsage)
+
+ remoteOnly := AdminMediaListRemoteOnlyFlag()
+ remoteOnlyUsage := fieldtag("AdminMediaListRemoteOnly", "usage")
+ cmd.Flags().Bool(remoteOnly, false, remoteOnlyUsage)
+}
+
// AddAdminMediaPrune attaches flags pertaining to media storage prune commands.
func AddAdminMediaPrune(cmd *cobra.Command) {
name := AdminMediaPruneDryRunFlag()
diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go
index 801d4f667..a515da200 100644
--- a/internal/config/helpers.gen.go
+++ b/internal/config/helpers.gen.go
@@ -3374,6 +3374,56 @@ func GetAdminMediaPruneDryRun() bool { return global.GetAdminMediaPruneDryRun()
// SetAdminMediaPruneDryRun safely sets the value for global configuration 'AdminMediaPruneDryRun' field
func SetAdminMediaPruneDryRun(v bool) { global.SetAdminMediaPruneDryRun(v) }
+// GetAdminMediaListLocalOnly safely fetches the Configuration value for state's 'AdminMediaListLocalOnly' field
+func (st *ConfigState) GetAdminMediaListLocalOnly() (v bool) {
+ st.mutex.RLock()
+ v = st.config.AdminMediaListLocalOnly
+ st.mutex.RUnlock()
+ return
+}
+
+// SetAdminMediaListLocalOnly safely sets the Configuration value for state's 'AdminMediaListLocalOnly' field
+func (st *ConfigState) SetAdminMediaListLocalOnly(v bool) {
+ st.mutex.Lock()
+ defer st.mutex.Unlock()
+ st.config.AdminMediaListLocalOnly = v
+ st.reloadToViper()
+}
+
+// AdminMediaListLocalOnlyFlag returns the flag name for the 'AdminMediaListLocalOnly' field
+func AdminMediaListLocalOnlyFlag() string { return "local-only" }
+
+// GetAdminMediaListLocalOnly safely fetches the value for global configuration 'AdminMediaListLocalOnly' field
+func GetAdminMediaListLocalOnly() bool { return global.GetAdminMediaListLocalOnly() }
+
+// SetAdminMediaListLocalOnly safely sets the value for global configuration 'AdminMediaListLocalOnly' field
+func SetAdminMediaListLocalOnly(v bool) { global.SetAdminMediaListLocalOnly(v) }
+
+// GetAdminMediaListRemoteOnly safely fetches the Configuration value for state's 'AdminMediaListRemoteOnly' field
+func (st *ConfigState) GetAdminMediaListRemoteOnly() (v bool) {
+ st.mutex.RLock()
+ v = st.config.AdminMediaListRemoteOnly
+ st.mutex.RUnlock()
+ return
+}
+
+// SetAdminMediaListRemoteOnly safely sets the Configuration value for state's 'AdminMediaListRemoteOnly' field
+func (st *ConfigState) SetAdminMediaListRemoteOnly(v bool) {
+ st.mutex.Lock()
+ defer st.mutex.Unlock()
+ st.config.AdminMediaListRemoteOnly = v
+ st.reloadToViper()
+}
+
+// AdminMediaListRemoteOnlyFlag returns the flag name for the 'AdminMediaListRemoteOnly' field
+func AdminMediaListRemoteOnlyFlag() string { return "remote-only" }
+
+// GetAdminMediaListRemoteOnly safely fetches the value for global configuration 'AdminMediaListRemoteOnly' field
+func GetAdminMediaListRemoteOnly() bool { return global.GetAdminMediaListRemoteOnly() }
+
+// SetAdminMediaListRemoteOnly safely sets the value for global configuration 'AdminMediaListRemoteOnly' field
+func SetAdminMediaListRemoteOnly(v bool) { global.SetAdminMediaListRemoteOnly(v) }
+
// GetRequestIDHeader safely fetches the Configuration value for state's 'RequestIDHeader' field
func (st *ConfigState) GetRequestIDHeader() (v string) {
st.mutex.RLock()
diff --git a/test/envparsing.sh b/test/envparsing.sh
index bd4e69309..63f1bc5e3 100755
--- a/test/envparsing.sh
+++ b/test/envparsing.sh
@@ -87,6 +87,7 @@ EXPECT=$(cat << "EOF"
"letsencrypt-email-address": "",
"letsencrypt-enabled": true,
"letsencrypt-port": 80,
+ "local-only": false,
"log-client-ip": false,
"log-db-queries": true,
"log-level": "info",
@@ -116,6 +117,7 @@ EXPECT=$(cat << "EOF"
"path": "",
"port": 6969,
"protocol": "http",
+ "remote-only": false,
"request-id-header": "X-Trace-Id",
"smtp-disclose-recipients": true,
"smtp-from": "queen.rip.in.piss@terfisland.org",