summaryrefslogtreecommitdiff
path: root/internal
diff options
context:
space:
mode:
Diffstat (limited to 'internal')
-rw-r--r--internal/config/config.go22
-rw-r--r--internal/config/config_test.go2
-rw-r--r--internal/config/defaults.go14
-rw-r--r--internal/config/helpers.gen.go148
-rw-r--r--internal/config/testdata/test3.yaml3
-rw-r--r--internal/middleware/nollamas.go4
-rw-r--r--internal/middleware/nollamas_test.go2
7 files changed, 127 insertions, 68 deletions
diff --git a/internal/config/config.go b/internal/config/config.go
index b69af5dda..0aa90ce44 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -254,13 +254,13 @@ type CacheConfiguration struct {
}
type AdvancedConfig struct {
- CookiesSamesite string `name:"cookies-samesite" usage:"'strict' or 'lax', see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie/SameSite"`
- SenderMultiplier int `name:"sender-multiplier" usage:"Multiplier to use per cpu for batching outgoing fedi messages. 0 or less turns batching off (not recommended)."`
- CSPExtraURIs []string `name:"csp-extra-uris" usage:"Additional URIs to allow when building content-security-policy for media + images."`
- HeaderFilterMode string `name:"header-filter-mode" usage:"Set incoming request header filtering mode."`
- ScraperDeterrence bool `name:"scraper-deterrence" usage:"Enable proof-of-work based scraper deterrence on profile / status pages"`
- RateLimit RateLimitConfig `name:"rate-limit"`
- Throttling ThrottlingConfig `name:"throttling"`
+ CookiesSamesite string `name:"cookies-samesite" usage:"'strict' or 'lax', see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie/SameSite"`
+ SenderMultiplier int `name:"sender-multiplier" usage:"Multiplier to use per cpu for batching outgoing fedi messages. 0 or less turns batching off (not recommended)."`
+ CSPExtraURIs []string `name:"csp-extra-uris" usage:"Additional URIs to allow when building content-security-policy for media + images."`
+ HeaderFilterMode string `name:"header-filter-mode" usage:"Set incoming request header filtering mode."`
+ RateLimit RateLimitConfig `name:"rate-limit"`
+ Throttling ThrottlingConfig `name:"throttling"`
+ ScraperDeterrence ScraperDeterrenceConfig `name:"scraper-deterrence"`
}
type RateLimitConfig struct {
@@ -273,7 +273,7 @@ type ThrottlingConfig struct {
RetryAfter time.Duration `name:"retry-after" usage:"Retry-After duration response to send for throttled requests."`
}
-// type ScraperDeterrenceConfig struct {
-// Enabled bool `name:"enabled" usage:"Enable proof-of-work based scraper deterrence on profile / status pages"`
-// Difficulty uint8 `name:"difficulty" usage:"The proof-of-work difficulty, which determines how many leading zeros to try solve in hash solutions."`
-// }
+type ScraperDeterrenceConfig struct {
+ Enabled bool `name:"enabled" usage:"Enable proof-of-work based scraper deterrence on profile / status pages"`
+ Difficulty uint8 `name:"difficulty" usage:"The proof-of-work difficulty, which determines how many leading zeros to try solve in hash solutions."`
+}
diff --git a/internal/config/config_test.go b/internal/config/config_test.go
index 48a138aa8..4e920700e 100644
--- a/internal/config/config_test.go
+++ b/internal/config/config_test.go
@@ -175,7 +175,7 @@ func TestCLIParsing(t *testing.T) {
"--config-path", "testdata/test3.yaml",
},
expected: expectedKV(
- kv.Field{"advanced-scraper-deterrence", true},
+ kv.Field{"advanced-scraper-deterrence-enabled", true},
kv.Field{"advanced-rate-limit-requests", 5000},
),
},
diff --git a/internal/config/defaults.go b/internal/config/defaults.go
index 6e4d7e09a..100738253 100644
--- a/internal/config/defaults.go
+++ b/internal/config/defaults.go
@@ -131,11 +131,10 @@ var Defaults = Configuration{
SyslogAddress: "localhost:514",
Advanced: AdvancedConfig{
- SenderMultiplier: 2, // 2 senders per CPU
- CSPExtraURIs: []string{},
- HeaderFilterMode: RequestHeaderFilterModeDisabled,
- CookiesSamesite: "lax",
- ScraperDeterrence: false,
+ SenderMultiplier: 2, // 2 senders per CPU
+ CSPExtraURIs: []string{},
+ HeaderFilterMode: RequestHeaderFilterModeDisabled,
+ CookiesSamesite: "lax",
RateLimit: RateLimitConfig{
Requests: 300, // 1 per second per 5 minutes
@@ -146,6 +145,11 @@ var Defaults = Configuration{
Multiplier: 8, // 8 open requests per CPU
RetryAfter: 30 * time.Second,
},
+
+ ScraperDeterrence: ScraperDeterrenceConfig{
+ Enabled: false,
+ Difficulty: 4,
+ },
},
Cache: CacheConfiguration{
diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go
index 5ca6a69e7..d38458304 100644
--- a/internal/config/helpers.gen.go
+++ b/internal/config/helpers.gen.go
@@ -136,11 +136,12 @@ func (cfg *Configuration) RegisterFlags(flags *pflag.FlagSet) {
flags.Int("advanced-sender-multiplier", cfg.Advanced.SenderMultiplier, "Multiplier to use per cpu for batching outgoing fedi messages. 0 or less turns batching off (not recommended).")
flags.StringSlice("advanced-csp-extra-uris", cfg.Advanced.CSPExtraURIs, "Additional URIs to allow when building content-security-policy for media + images.")
flags.String("advanced-header-filter-mode", cfg.Advanced.HeaderFilterMode, "Set incoming request header filtering mode.")
- flags.Bool("advanced-scraper-deterrence", cfg.Advanced.ScraperDeterrence, "Enable proof-of-work based scraper deterrence on profile / status pages")
flags.Int("advanced-rate-limit-requests", cfg.Advanced.RateLimit.Requests, "Amount of HTTP requests to permit within a 5 minute window. 0 or less turns rate limiting off.")
flags.StringSlice("advanced-rate-limit-exceptions", cfg.Advanced.RateLimit.Exceptions.Strings(), "Slice of CIDRs to exclude from rate limit restrictions.")
flags.Int("advanced-throttling-multiplier", cfg.Advanced.Throttling.Multiplier, "Multiplier to use per cpu for http request throttling. 0 or less turns throttling off.")
flags.Duration("advanced-throttling-retry-after", cfg.Advanced.Throttling.RetryAfter, "Retry-After duration response to send for throttled requests.")
+ flags.Bool("advanced-scraper-deterrence-enabled", cfg.Advanced.ScraperDeterrence.Enabled, "Enable proof-of-work based scraper deterrence on profile / status pages")
+ flags.Uint8("advanced-scraper-deterrence-difficulty", cfg.Advanced.ScraperDeterrence.Difficulty, "The proof-of-work difficulty, which determines how many leading zeros to try solve in hash solutions.")
flags.StringSlice("http-client-allow-ips", cfg.HTTPClient.AllowIPs, "")
flags.StringSlice("http-client-block-ips", cfg.HTTPClient.BlockIPs, "")
flags.Duration("http-client-timeout", cfg.HTTPClient.Timeout, "")
@@ -205,7 +206,7 @@ func (cfg *Configuration) RegisterFlags(flags *pflag.FlagSet) {
}
func (cfg *Configuration) MarshalMap() map[string]any {
- cfgmap := make(map[string]any, 180)
+ cfgmap := make(map[string]any, 181)
cfgmap["log-level"] = cfg.LogLevel
cfgmap["log-timestamp-format"] = cfg.LogTimestampFormat
cfgmap["log-db-queries"] = cfg.LogDbQueries
@@ -313,11 +314,12 @@ func (cfg *Configuration) MarshalMap() map[string]any {
cfgmap["advanced-sender-multiplier"] = cfg.Advanced.SenderMultiplier
cfgmap["advanced-csp-extra-uris"] = cfg.Advanced.CSPExtraURIs
cfgmap["advanced-header-filter-mode"] = cfg.Advanced.HeaderFilterMode
- cfgmap["advanced-scraper-deterrence"] = cfg.Advanced.ScraperDeterrence
cfgmap["advanced-rate-limit-requests"] = cfg.Advanced.RateLimit.Requests
cfgmap["advanced-rate-limit-exceptions"] = cfg.Advanced.RateLimit.Exceptions.Strings()
cfgmap["advanced-throttling-multiplier"] = cfg.Advanced.Throttling.Multiplier
cfgmap["advanced-throttling-retry-after"] = cfg.Advanced.Throttling.RetryAfter
+ cfgmap["advanced-scraper-deterrence-enabled"] = cfg.Advanced.ScraperDeterrence.Enabled
+ cfgmap["advanced-scraper-deterrence-difficulty"] = cfg.Advanced.ScraperDeterrence.Difficulty
cfgmap["http-client-allow-ips"] = cfg.HTTPClient.AllowIPs
cfgmap["http-client-block-ips"] = cfg.HTTPClient.BlockIPs
cfgmap["http-client-timeout"] = cfg.HTTPClient.Timeout
@@ -1277,14 +1279,6 @@ func (cfg *Configuration) UnmarshalMap(cfgmap map[string]any) error {
}
}
- if ival, ok := cfgmap["advanced-scraper-deterrence"]; ok {
- var err error
- cfg.Advanced.ScraperDeterrence, err = cast.ToBoolE(ival)
- if err != nil {
- return fmt.Errorf("error casting %#v -> bool for 'advanced-scraper-deterrence': %w", ival, err)
- }
- }
-
if ival, ok := cfgmap["advanced-rate-limit-requests"]; ok {
var err error
cfg.Advanced.RateLimit.Requests, err = cast.ToIntE(ival)
@@ -1322,6 +1316,22 @@ func (cfg *Configuration) UnmarshalMap(cfgmap map[string]any) error {
}
}
+ if ival, ok := cfgmap["advanced-scraper-deterrence-enabled"]; ok {
+ var err error
+ cfg.Advanced.ScraperDeterrence.Enabled, err = cast.ToBoolE(ival)
+ if err != nil {
+ return fmt.Errorf("error casting %#v -> bool for 'advanced-scraper-deterrence-enabled': %w", ival, err)
+ }
+ }
+
+ if ival, ok := cfgmap["advanced-scraper-deterrence-difficulty"]; ok {
+ var err error
+ cfg.Advanced.ScraperDeterrence.Difficulty, err = cast.ToUint8E(ival)
+ if err != nil {
+ return fmt.Errorf("error casting %#v -> uint8 for 'advanced-scraper-deterrence-difficulty': %w", ival, err)
+ }
+ }
+
if ival, ok := cfgmap["http-client-allow-ips"]; ok {
var err error
cfg.HTTPClient.AllowIPs, err = toStringSlice(ival)
@@ -4553,31 +4563,6 @@ func GetAdvancedHeaderFilterMode() string { return global.GetAdvancedHeaderFilte
// SetAdvancedHeaderFilterMode safely sets the value for global configuration 'Advanced.HeaderFilterMode' field
func SetAdvancedHeaderFilterMode(v string) { global.SetAdvancedHeaderFilterMode(v) }
-// AdvancedScraperDeterrenceFlag returns the flag name for the 'Advanced.ScraperDeterrence' field
-func AdvancedScraperDeterrenceFlag() string { return "advanced-scraper-deterrence" }
-
-// GetAdvancedScraperDeterrence safely fetches the Configuration value for state's 'Advanced.ScraperDeterrence' field
-func (st *ConfigState) GetAdvancedScraperDeterrence() (v bool) {
- st.mutex.RLock()
- v = st.config.Advanced.ScraperDeterrence
- st.mutex.RUnlock()
- return
-}
-
-// SetAdvancedScraperDeterrence safely sets the Configuration value for state's 'Advanced.ScraperDeterrence' field
-func (st *ConfigState) SetAdvancedScraperDeterrence(v bool) {
- st.mutex.Lock()
- defer st.mutex.Unlock()
- st.config.Advanced.ScraperDeterrence = v
- st.reloadToViper()
-}
-
-// GetAdvancedScraperDeterrence safely fetches the value for global configuration 'Advanced.ScraperDeterrence' field
-func GetAdvancedScraperDeterrence() bool { return global.GetAdvancedScraperDeterrence() }
-
-// SetAdvancedScraperDeterrence safely sets the value for global configuration 'Advanced.ScraperDeterrence' field
-func SetAdvancedScraperDeterrence(v bool) { global.SetAdvancedScraperDeterrence(v) }
-
// AdvancedRateLimitRequestsFlag returns the flag name for the 'Advanced.RateLimit.Requests' field
func AdvancedRateLimitRequestsFlag() string { return "advanced-rate-limit-requests" }
@@ -4678,6 +4663,62 @@ func GetAdvancedThrottlingRetryAfter() time.Duration { return global.GetAdvanced
// SetAdvancedThrottlingRetryAfter safely sets the value for global configuration 'Advanced.Throttling.RetryAfter' field
func SetAdvancedThrottlingRetryAfter(v time.Duration) { global.SetAdvancedThrottlingRetryAfter(v) }
+// AdvancedScraperDeterrenceEnabledFlag returns the flag name for the 'Advanced.ScraperDeterrence.Enabled' field
+func AdvancedScraperDeterrenceEnabledFlag() string { return "advanced-scraper-deterrence-enabled" }
+
+// GetAdvancedScraperDeterrenceEnabled safely fetches the Configuration value for state's 'Advanced.ScraperDeterrence.Enabled' field
+func (st *ConfigState) GetAdvancedScraperDeterrenceEnabled() (v bool) {
+ st.mutex.RLock()
+ v = st.config.Advanced.ScraperDeterrence.Enabled
+ st.mutex.RUnlock()
+ return
+}
+
+// SetAdvancedScraperDeterrenceEnabled safely sets the Configuration value for state's 'Advanced.ScraperDeterrence.Enabled' field
+func (st *ConfigState) SetAdvancedScraperDeterrenceEnabled(v bool) {
+ st.mutex.Lock()
+ defer st.mutex.Unlock()
+ st.config.Advanced.ScraperDeterrence.Enabled = v
+ st.reloadToViper()
+}
+
+// GetAdvancedScraperDeterrenceEnabled safely fetches the value for global configuration 'Advanced.ScraperDeterrence.Enabled' field
+func GetAdvancedScraperDeterrenceEnabled() bool { return global.GetAdvancedScraperDeterrenceEnabled() }
+
+// SetAdvancedScraperDeterrenceEnabled safely sets the value for global configuration 'Advanced.ScraperDeterrence.Enabled' field
+func SetAdvancedScraperDeterrenceEnabled(v bool) { global.SetAdvancedScraperDeterrenceEnabled(v) }
+
+// AdvancedScraperDeterrenceDifficultyFlag returns the flag name for the 'Advanced.ScraperDeterrence.Difficulty' field
+func AdvancedScraperDeterrenceDifficultyFlag() string {
+ return "advanced-scraper-deterrence-difficulty"
+}
+
+// GetAdvancedScraperDeterrenceDifficulty safely fetches the Configuration value for state's 'Advanced.ScraperDeterrence.Difficulty' field
+func (st *ConfigState) GetAdvancedScraperDeterrenceDifficulty() (v uint8) {
+ st.mutex.RLock()
+ v = st.config.Advanced.ScraperDeterrence.Difficulty
+ st.mutex.RUnlock()
+ return
+}
+
+// SetAdvancedScraperDeterrenceDifficulty safely sets the Configuration value for state's 'Advanced.ScraperDeterrence.Difficulty' field
+func (st *ConfigState) SetAdvancedScraperDeterrenceDifficulty(v uint8) {
+ st.mutex.Lock()
+ defer st.mutex.Unlock()
+ st.config.Advanced.ScraperDeterrence.Difficulty = v
+ st.reloadToViper()
+}
+
+// GetAdvancedScraperDeterrenceDifficulty safely fetches the value for global configuration 'Advanced.ScraperDeterrence.Difficulty' field
+func GetAdvancedScraperDeterrenceDifficulty() uint8 {
+ return global.GetAdvancedScraperDeterrenceDifficulty()
+}
+
+// SetAdvancedScraperDeterrenceDifficulty safely sets the value for global configuration 'Advanced.ScraperDeterrence.Difficulty' field
+func SetAdvancedScraperDeterrenceDifficulty(v uint8) {
+ global.SetAdvancedScraperDeterrenceDifficulty(v)
+}
+
// HTTPClientAllowIPsFlag returns the flag name for the 'HTTPClient.AllowIPs' field
func HTTPClientAllowIPsFlag() string { return "http-client-allow-ips" }
@@ -6451,17 +6492,6 @@ func flattenConfigMap(cfgmap map[string]any) {
}
for _, key := range [][]string{
- {"advanced", "scraper-deterrence"},
- } {
- ival, ok := mapGet(cfgmap, key...)
- if ok {
- cfgmap["advanced-scraper-deterrence"] = ival
- nestedKeys[key[0]] = struct{}{}
- break
- }
- }
-
- for _, key := range [][]string{
{"advanced-rate-limit", "requests"},
{"advanced", "rate-limit", "requests"},
} {
@@ -6510,6 +6540,30 @@ func flattenConfigMap(cfgmap map[string]any) {
}
for _, key := range [][]string{
+ {"advanced-scraper-deterrence", "enabled"},
+ {"advanced", "scraper-deterrence", "enabled"},
+ } {
+ ival, ok := mapGet(cfgmap, key...)
+ if ok {
+ cfgmap["advanced-scraper-deterrence-enabled"] = ival
+ nestedKeys[key[0]] = struct{}{}
+ break
+ }
+ }
+
+ for _, key := range [][]string{
+ {"advanced-scraper-deterrence", "difficulty"},
+ {"advanced", "scraper-deterrence", "difficulty"},
+ } {
+ ival, ok := mapGet(cfgmap, key...)
+ if ok {
+ cfgmap["advanced-scraper-deterrence-difficulty"] = ival
+ nestedKeys[key[0]] = struct{}{}
+ break
+ }
+ }
+
+ for _, key := range [][]string{
{"http-client", "allow-ips"},
} {
ival, ok := mapGet(cfgmap, key...)
diff --git a/internal/config/testdata/test3.yaml b/internal/config/testdata/test3.yaml
index b51c3c3d0..d383fdd80 100644
--- a/internal/config/testdata/test3.yaml
+++ b/internal/config/testdata/test3.yaml
@@ -1,4 +1,5 @@
advanced:
- scraper-deterrence: true
+ scraper-deterrence:
+ enabled: true
rate-limit:
requests: 5000
diff --git a/internal/middleware/nollamas.go b/internal/middleware/nollamas.go
index eedf3b9b7..914469a24 100644
--- a/internal/middleware/nollamas.go
+++ b/internal/middleware/nollamas.go
@@ -55,7 +55,7 @@ func NoLLaMas(
getInstanceV1 func(context.Context) (*apimodel.InstanceV1, gtserror.WithCode),
) gin.HandlerFunc {
- if !config.GetAdvancedScraperDeterrence() {
+ if !config.GetAdvancedScraperDeterrenceEnabled() {
// NoLLaMas middleware disabled.
return func(*gin.Context) {}
}
@@ -72,7 +72,7 @@ func NoLLaMas(
var nollamas nollamas
nollamas.seed = seed
nollamas.ttl = time.Hour
- nollamas.diff = 4
+ nollamas.diff = config.GetAdvancedScraperDeterrenceDifficulty()
nollamas.getInstanceV1 = getInstanceV1
nollamas.policy = cookiePolicy
return nollamas.Serve
diff --git a/internal/middleware/nollamas_test.go b/internal/middleware/nollamas_test.go
index 37b66e5e9..ffe8cbbc8 100644
--- a/internal/middleware/nollamas_test.go
+++ b/internal/middleware/nollamas_test.go
@@ -45,7 +45,7 @@ func TestNoLLaMasMiddleware(t *testing.T) {
e := gin.New()
// Setup necessary configuration variables.
- config.SetAdvancedScraperDeterrence(true)
+ config.SetAdvancedScraperDeterrenceEnabled(true)
config.SetWebTemplateBaseDir("../../web/template")
// Load templates into engine.
generated by cgit v1.2.3 (git 2.46.0) at 2025-12-02 20:18:43 +0000