[feature] support configuring database caches (#1246)

* update config generator to support nested structs, add cache configuration options

* update envparsing test

* add cache configuration to config parse tests

* set cache configuration in testrig

* move caches to sub-cache "gts" namespace, update envparsing, add cache config docs to example config

Signed-off-by: kim <grufwub@gmail.com>

1 files changed, 201 insertions, 48 deletions

diff --git a/internal/config/gen/gen.go b/internal/config/gen/gen.go
index a0db7e0e0..9065528e6 100644
--- a/internal/config/gen/gen.go
+++ b/internal/config/gen/gen.go
@@ -21,11 +21,14 @@ package main
 import (
 	"flag"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"reflect"
+	"strings"
+	"time"
 
-	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"codeberg.org/gruf/go-bytesize"
 )
 
 const license = `/*
@@ -47,15 +50,154 @@ const license = `/*
 */
 `
 
+type Configuration struct {
+	LogLevel        string   `name:"log-level" usage:"Log level to run at: [trace, debug, info, warn, fatal]"`
+	LogDbQueries    bool     `name:"log-db-queries" usage:"Log database queries verbosely when log-level is trace or debug"`
+	ApplicationName string   `name:"application-name" usage:"Name of the application, used in various places internally"`
+	LandingPageUser string   `name:"landing-page-user" usage:"the user that should be shown on the instance's landing page"`
+	ConfigPath      string   `name:"config-path" usage:"Path to a file containing gotosocial configuration. Values set in this file will be overwritten by values set as env vars or arguments"`
+	Host            string   `name:"host" usage:"Hostname to use for the server (eg., example.org, gotosocial.whatever.com). DO NOT change this on a server that's already run!"`
+	AccountDomain   string   `name:"account-domain" usage:"Domain to use in account names (eg., example.org, whatever.com). If not set, will default to the setting for host. DO NOT change this on a server that's already run!"`
+	Protocol        string   `name:"protocol" usage:"Protocol to use for the REST api of the server (only use http if you are debugging or behind a reverse proxy!)"`
+	BindAddress     string   `name:"bind-address" usage:"Bind address to use for the GoToSocial server (eg., 0.0.0.0, 172.138.0.9, [::], localhost). For ipv6, enclose the address in square brackets, eg [2001:db8::fed1]. Default binds to all interfaces."`
+	Port            int      `name:"port" usage:"Port to use for GoToSocial. Change this to 443 if you're running the binary directly on the host machine."`
+	TrustedProxies  []string `name:"trusted-proxies" usage:"Proxies to trust when parsing x-forwarded headers into real IPs."`
+	SoftwareVersion string   `name:"software-version" usage:""`
+
+	DbType      string `name:"db-type" usage:"Database type: eg., postgres"`
+	DbAddress   string `name:"db-address" usage:"Database ipv4 address, hostname, or filename"`
+	DbPort      int    `name:"db-port" usage:"Database port"`
+	DbUser      string `name:"db-user" usage:"Database username"`
+	DbPassword  string `name:"db-password" usage:"Database password"`
+	DbDatabase  string `name:"db-database" usage:"Database name"`
+	DbTLSMode   string `name:"db-tls-mode" usage:"Database tls mode"`
+	DbTLSCACert string `name:"db-tls-ca-cert" usage:"Path to CA cert for db tls connection"`
+
+	WebTemplateBaseDir string `name:"web-template-base-dir" usage:"Basedir for html templating files for rendering pages and composing emails."`
+	WebAssetBaseDir    string `name:"web-asset-base-dir" usage:"Directory to serve static assets from, accessible at example.org/assets/"`
+
+	InstanceExposePeers            bool `name:"instance-expose-peers" usage:"Allow unauthenticated users to query /api/v1/instance/peers?filter=open"`
+	InstanceExposeSuspended        bool `name:"instance-expose-suspended" usage:"Expose suspended instances via web UI, and allow unauthenticated users to query /api/v1/instance/peers?filter=suspended"`
+	InstanceExposePublicTimeline   bool `name:"instance-expose-public-timeline" usage:"Allow unauthenticated users to query /api/v1/timelines/public"`
+	InstanceDeliverToSharedInboxes bool `name:"instance-deliver-to-shared-inboxes" usage:"Deliver federated messages to shared inboxes, if they're available."`
+
+	AccountsRegistrationOpen bool `name:"accounts-registration-open" usage:"Allow anyone to submit an account signup request. If false, server will be invite-only."`
+	AccountsApprovalRequired bool `name:"accounts-approval-required" usage:"Do account signups require approval by an admin or moderator before user can log in? If false, new registrations will be automatically approved."`
+	AccountsReasonRequired   bool `name:"accounts-reason-required" usage:"Do new account signups require a reason to be submitted on registration?"`
+	AccountsAllowCustomCSS   bool `name:"accounts-allow-custom-css" usage:"Allow accounts to enable custom CSS for their profile pages and statuses."`
+
+	MediaImageMaxSize        bytesize.Size `name:"media-image-max-size" usage:"Max size of accepted images in bytes"`
+	MediaVideoMaxSize        bytesize.Size `name:"media-video-max-size" usage:"Max size of accepted videos in bytes"`
+	MediaDescriptionMinChars int           `name:"media-description-min-chars" usage:"Min required chars for an image description"`
+	MediaDescriptionMaxChars int           `name:"media-description-max-chars" usage:"Max permitted chars for an image description"`
+	MediaRemoteCacheDays     int           `name:"media-remote-cache-days" usage:"Number of days to locally cache media from remote instances. If set to 0, remote media will be kept indefinitely."`
+	MediaEmojiLocalMaxSize   bytesize.Size `name:"media-emoji-local-max-size" usage:"Max size in bytes of emojis uploaded to this instance via the admin API."`
+	MediaEmojiRemoteMaxSize  bytesize.Size `name:"media-emoji-remote-max-size" usage:"Max size in bytes of emojis to download from other instances."`
+
+	StorageBackend       string `name:"storage-backend" usage:"Storage backend to use for media attachments"`
+	StorageLocalBasePath string `name:"storage-local-base-path" usage:"Full path to an already-created directory where gts should store/retrieve media files. Subfolders will be created within this dir."`
+	StorageS3Endpoint    string `name:"storage-s3-endpoint" usage:"S3 Endpoint URL (e.g 'minio.example.org:9000')"`
+	StorageS3AccessKey   string `name:"storage-s3-access-key" usage:"S3 Access Key"`
+	StorageS3SecretKey   string `name:"storage-s3-secret-key" usage:"S3 Secret Key"`
+	StorageS3UseSSL      bool   `name:"storage-s3-use-ssl" usage:"Use SSL for S3 connections. Only set this to 'false' when testing locally"`
+	StorageS3BucketName  string `name:"storage-s3-bucket" usage:"Place blobs in this bucket"`
+	StorageS3Proxy       bool   `name:"storage-s3-proxy" usage:"Proxy S3 contents through GoToSocial instead of redirecting to a presigned URL"`
+
+	StatusesMaxChars           int `name:"statuses-max-chars" usage:"Max permitted characters for posted statuses"`
+	StatusesCWMaxChars         int `name:"statuses-cw-max-chars" usage:"Max permitted characters for content/spoiler warnings on statuses"`
+	StatusesPollMaxOptions     int `name:"statuses-poll-max-options" usage:"Max amount of options permitted on a poll"`
+	StatusesPollOptionMaxChars int `name:"statuses-poll-option-max-chars" usage:"Max amount of characters for a poll option"`
+	StatusesMediaMaxFiles      int `name:"statuses-media-max-files" usage:"Maximum number of media files/attachments per status"`
+
+	LetsEncryptEnabled      bool   `name:"letsencrypt-enabled" usage:"Enable letsencrypt TLS certs for this server. If set to true, then cert dir also needs to be set (or take the default)."`
+	LetsEncryptPort         int    `name:"letsencrypt-port" usage:"Port to listen on for letsencrypt certificate challenges. Must not be the same as the GtS webserver/API port."`
+	LetsEncryptCertDir      string `name:"letsencrypt-cert-dir" usage:"Directory to store acquired letsencrypt certificates."`
+	LetsEncryptEmailAddress string `name:"letsencrypt-email-address" usage:"Email address to use when requesting letsencrypt certs. Will receive updates on cert expiry etc."`
+
+	OIDCEnabled          bool     `name:"oidc-enabled" usage:"Enabled OIDC authorization for this instance. If set to true, then the other OIDC flags must also be set."`
+	OIDCIdpName          string   `name:"oidc-idp-name" usage:"Name of the OIDC identity provider. Will be shown to the user when logging in."`
+	OIDCSkipVerification bool     `name:"oidc-skip-verification" usage:"Skip verification of tokens returned by the OIDC provider. Should only be set to 'true' for testing purposes, never in a production environment!"`
+	OIDCIssuer           string   `name:"oidc-issuer" usage:"Address of the OIDC issuer. Should be the web address, including protocol, at which the issuer can be reached. Eg., 'https://example.org/auth'"`
+	OIDCClientID         string   `name:"oidc-client-id" usage:"ClientID of GoToSocial, as registered with the OIDC provider."`
+	OIDCClientSecret     string   `name:"oidc-client-secret" usage:"ClientSecret of GoToSocial, as registered with the OIDC provider."`
+	OIDCScopes           []string `name:"oidc-scopes" usage:"OIDC scopes."`
+	OIDCLinkExisting     bool     `name:"oidc-link-existing" usage:"link existing user accounts to OIDC logins based on the stored email value"`
+
+	SMTPHost     string `name:"smtp-host" usage:"Host of the smtp server. Eg., 'smtp.eu.mailgun.org'"`
+	SMTPPort     int    `name:"smtp-port" usage:"Port of the smtp server. Eg., 587"`
+	SMTPUsername string `name:"smtp-username" usage:"Username to authenticate with the smtp server as. Eg., 'postmaster@mail.example.org'"`
+	SMTPPassword string `name:"smtp-password" usage:"Password to pass to the smtp server."`
+	SMTPFrom     string `name:"smtp-from" usage:"Address to use as the 'from' field of the email. Eg., 'gotosocial@example.org'"`
+
+	SyslogEnabled  bool   `name:"syslog-enabled" usage:"Enable the syslog logging hook. Logs will be mirrored to the configured destination."`
+	SyslogProtocol string `name:"syslog-protocol" usage:"Protocol to use when directing logs to syslog. Leave empty to connect to local syslog."`
+	SyslogAddress  string `name:"syslog-address" usage:"Address:port to send syslog logs to. Leave empty to connect to local syslog."`
+
+	AdvancedCookiesSamesite   string `name:"advanced-cookies-samesite" usage:"'strict' or 'lax', see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie/SameSite"`
+	AdvancedRateLimitRequests int    `name:"advanced-rate-limit-requests" usage:"Amount of HTTP requests to permit within a 5 minute window. 0 or less turns rate limiting off."`
+
+	// Cache configuration vars.
+	Cache CacheConfiguration `name:"cache"`
+
+	// TODO: move these elsewhere, these are more ephemeral vs long-running flags like above
+	AdminAccountUsername  string `name:"username" usage:"the username to create/delete/etc"`
+	AdminAccountEmail     string `name:"email" usage:"the email address of this account"`
+	AdminAccountPassword  string `name:"password" usage:"the password to set for this account"`
+	AdminTransPath        string `name:"path" usage:"the path of the file to import from/export to"`
+	AdminMediaPruneDryRun bool   `name:"dry-run" usage:"perform a dry run and only log number of items eligible for pruning"`
+}
+
+type CacheConfiguration struct {
+	GTS GTSCacheConfiguration `name:"gts"`
+}
+
+type GTSCacheConfiguration struct {
+	AccountMaxSize   int           `name:"account-max-size"`
+	AccountTTL       time.Duration `name:"account-ttl"`
+	AccountSweepFreq time.Duration `name:"account-sweep-freq"`
+
+	BlockMaxSize   int           `name:"block-max-size"`
+	BlockTTL       time.Duration `name:"block-ttl"`
+	BlockSweepFreq time.Duration `name:"block-sweep-freq"`
+
+	DomainBlockMaxSize   int           `name:"domain-block-max-size"`
+	DomainBlockTTL       time.Duration `name:"domain-block-ttl"`
+	DomainBlockSweepFreq time.Duration `name:"domain-block-sweep-freq"`
+
+	EmojiMaxSize   int           `name:"emoji-max-size"`
+	EmojiTTL       time.Duration `name:"emoji-ttl"`
+	EmojiSweepFreq time.Duration `name:"emoji-sweep-freq"`
+
+	EmojiCategoryMaxSize   int           `name:"emoji-category-max-size"`
+	EmojiCategoryTTL       time.Duration `name:"emoji-category-ttl"`
+	EmojiCategorySweepFreq time.Duration `name:"emoji-category-sweep-freq"`
+
+	MentionMaxSize   int           `name:"mention-max-size"`
+	MentionTTL       time.Duration `name:"mention-ttl"`
+	MentionSweepFreq time.Duration `name:"mention-sweep-freq"`
+
+	NotificationMaxSize   int           `name:"notification-max-size"`
+	NotificationTTL       time.Duration `name:"notification-ttl"`
+	NotificationSweepFreq time.Duration `name:"notification-sweep-freq"`
+
+	StatusMaxSize   int           `name:"status-max-size"`
+	StatusTTL       time.Duration `name:"status-ttl"`
+	StatusSweepFreq time.Duration `name:"status-sweep-freq"`
+
+	TombstoneMaxSize   int           `name:"tombstone-max-size"`
+	TombstoneTTL       time.Duration `name:"tombstone-ttl"`
+	TombstoneSweepFreq time.Duration `name:"tombstone-sweep-freq"`
+
+	UserMaxSize   int           `name:"user-max-size"`
+	UserTTL       time.Duration `name:"user-ttl"`
+	UserSweepFreq time.Duration `name:"user-sweep-freq"`
+}
+
 func main() {
-	var (
-		out string
-		gen string
-	)
+	var out string
 
 	// Load runtime config flags
 	flag.StringVar(&out, "out", "", "Generated file output path")
-	flag.StringVar(&gen, "gen", "helpers", "Type of file to generate (helpers)")
 	flag.Parse()
 
 	// Open output file path
@@ -64,50 +206,61 @@ func main() {
 		panic(err)
 	}
 
-	switch gen {
-	// Generate config field helper methods
-	case "helpers":
-		fmt.Fprint(output, "// THIS IS A GENERATED FILE, DO NOT EDIT BY HAND\n")
-		fmt.Fprint(output, license)
-		fmt.Fprint(output, "package config\n\n")
-		fmt.Fprint(output, "import \"codeberg.org/gruf/go-bytesize\"\n\n")
-		t := reflect.TypeOf(config.Configuration{})
-		for i := 0; i < t.NumField(); i++ {
-			field := t.Field(i)
-
-			// ConfigState structure helper methods
-			fmt.Fprintf(output, "// Get%s safely fetches the Configuration value for state's '%s' field\n", field.Name, field.Name)
-			fmt.Fprintf(output, "func (st *ConfigState) Get%s() (v %s) {\n", field.Name, field.Type.String())
-			fmt.Fprintf(output, "\tst.mutex.Lock()\n")
-			fmt.Fprintf(output, "\tv = st.config.%s\n", field.Name)
-			fmt.Fprintf(output, "\tst.mutex.Unlock()\n")
-			fmt.Fprintf(output, "\treturn\n")
-			fmt.Fprintf(output, "}\n\n")
-			fmt.Fprintf(output, "// Set%s safely sets the Configuration value for state's '%s' field\n", field.Name, field.Name)
-			fmt.Fprintf(output, "func (st *ConfigState) Set%s(v %s) {\n", field.Name, field.Type.String())
-			fmt.Fprintf(output, "\tst.mutex.Lock()\n")
-			fmt.Fprintf(output, "\tdefer st.mutex.Unlock()\n")
-			fmt.Fprintf(output, "\tst.config.%s = v\n", field.Name)
-			fmt.Fprintf(output, "\tst.reloadToViper()\n")
-			fmt.Fprintf(output, "}\n\n")
-
-			// Global ConfigState helper methods
-			// TODO: remove when we pass around a ConfigState{}
-			fmt.Fprintf(output, "// %sFlag returns the flag name for the '%s' field\n", field.Name, field.Name)
-			fmt.Fprintf(output, "func %sFlag() string { return \"%s\" }\n\n", field.Name, field.Tag.Get("name"))
-			fmt.Fprintf(output, "// Get%s safely fetches the value for global configuration '%s' field\n", field.Name, field.Name)
-			fmt.Fprintf(output, "func Get%[1]s() %[2]s { return global.Get%[1]s() }\n\n", field.Name, field.Type.String())
-			fmt.Fprintf(output, "// Set%s safely sets the value for global configuration '%s' field\n", field.Name, field.Name)
-			fmt.Fprintf(output, "func Set%[1]s(v %[2]s) { global.Set%[1]s(v) }\n\n", field.Name, field.Type.String())
-		}
-		_ = output.Close()
-		_ = exec.Command("gofumports", "-w", out).Run()
+	fmt.Fprint(output, "// THIS IS A GENERATED FILE, DO NOT EDIT BY HAND\n")
+	fmt.Fprint(output, license)
+	fmt.Fprint(output, "package config\n\n")
+	fmt.Fprint(output, "import \"codeberg.org/gruf/go-bytesize\"\n\n")
+	generateFields(output, nil, reflect.TypeOf(Configuration{}))
+	_ = output.Close()
+	_ = exec.Command("gofumports", "-w", out).Run()
 
-	// The plain here is that eventually we might be able
+	// The plan here is that eventually we might be able
 	// to generate an example configuration from struct tags
+}
+
+func generateFields(output io.Writer, prefixes []string, t reflect.Type) {
+	for i := 0; i < t.NumField(); i++ {
+		field := t.Field(i)
+
+		if ft := field.Type; ft.Kind() == reflect.Struct {
+			// This is a struct field containing further nested config vars.
+			generateFields(output, append(prefixes, field.Name), ft)
+			continue
+		}
+
+		// Get prefixed config variable name
+		name := strings.Join(prefixes, "") + field.Name
+
+		// Get period-separated (if nested) config variable "path"
+		fieldPath := strings.Join(append(prefixes, field.Name), ".")
+
+		// Get dash-separated config variable CLI flag "path"
+		flagPath := strings.Join(append(prefixes, field.Tag.Get("name")), "-")
+		flagPath = strings.ToLower(flagPath)
+
+		// ConfigState structure helper methods
+		fmt.Fprintf(output, "// Get%s safely fetches the Configuration value for state's '%s' field\n", name, fieldPath)
+		fmt.Fprintf(output, "func (st *ConfigState) Get%s() (v %s) {\n", name, field.Type.String())
+		fmt.Fprintf(output, "\tst.mutex.Lock()\n")
+		fmt.Fprintf(output, "\tv = st.config.%s\n", fieldPath)
+		fmt.Fprintf(output, "\tst.mutex.Unlock()\n")
+		fmt.Fprintf(output, "\treturn\n")
+		fmt.Fprintf(output, "}\n\n")
+		fmt.Fprintf(output, "// Set%s safely sets the Configuration value for state's '%s' field\n", name, fieldPath)
+		fmt.Fprintf(output, "func (st *ConfigState) Set%s(v %s) {\n", name, field.Type.String())
+		fmt.Fprintf(output, "\tst.mutex.Lock()\n")
+		fmt.Fprintf(output, "\tdefer st.mutex.Unlock()\n")
+		fmt.Fprintf(output, "\tst.config.%s = v\n", fieldPath)
+		fmt.Fprintf(output, "\tst.reloadToViper()\n")
+		fmt.Fprintf(output, "}\n\n")
 
-	// Unknown type
-	default:
-		panic("unknown generation type: " + gen)
+		// Global ConfigState helper methods
+		// TODO: remove when we pass around a ConfigState{}
+		fmt.Fprintf(output, "// %sFlag returns the flag name for the '%s' field\n", name, fieldPath)
+		fmt.Fprintf(output, "func %sFlag() string { return \"%s\" }\n\n", name, flagPath)
+		fmt.Fprintf(output, "// Get%s safely fetches the value for global configuration '%s' field\n", name, fieldPath)
+		fmt.Fprintf(output, "func Get%[1]s() %[2]s { return global.Get%[1]s() }\n\n", name, field.Type.String())
+		fmt.Fprintf(output, "// Set%s safely sets the value for global configuration '%s' field\n", name, fieldPath)
+		fmt.Fprintf(output, "func Set%[1]s(v %[2]s) { global.Set%[1]s(v) }\n\n", name, field.Type.String())
 	}
 }