diff options
54 files changed, 1844 insertions, 2671 deletions
diff --git a/cmd/gotosocial/action/admin/account/account.go b/cmd/gotosocial/action/admin/account/account.go index ad8da5d41..d7cd40df4 100644 --- a/cmd/gotosocial/action/admin/account/account.go +++ b/cmd/gotosocial/action/admin/account/account.go @@ -36,6 +36,7 @@ import ( var Create action.GTSAction = func(ctx context.Context) error { var state state.State state.Caches.Init() + state.Workers.Start() dbConn, err := bundb.NewBunDBService(ctx, &state) if err != nil { @@ -97,6 +98,7 @@ var Create action.GTSAction = func(ctx context.Context) error { var Confirm action.GTSAction = func(ctx context.Context) error { var state state.State state.Caches.Init() + state.Workers.Start() dbConn, err := bundb.NewBunDBService(ctx, &state) if err != nil { @@ -140,6 +142,7 @@ var Confirm action.GTSAction = func(ctx context.Context) error { var Promote action.GTSAction = func(ctx context.Context) error { var state state.State state.Caches.Init() + state.Workers.Start() dbConn, err := bundb.NewBunDBService(ctx, &state) if err != nil { @@ -180,6 +183,7 @@ var Promote action.GTSAction = func(ctx context.Context) error { var Demote action.GTSAction = func(ctx context.Context) error { var state state.State state.Caches.Init() + state.Workers.Start() dbConn, err := bundb.NewBunDBService(ctx, &state) if err != nil { @@ -220,6 +224,7 @@ var Demote action.GTSAction = func(ctx context.Context) error { var Disable action.GTSAction = func(ctx context.Context) error { var state state.State state.Caches.Init() + state.Workers.Start() dbConn, err := bundb.NewBunDBService(ctx, &state) if err != nil { @@ -260,6 +265,7 @@ var Disable action.GTSAction = func(ctx context.Context) error { var Password action.GTSAction = func(ctx context.Context) error { var state state.State state.Caches.Init() + state.Workers.Start() dbConn, err := bundb.NewBunDBService(ctx, &state) if err != nil { diff --git a/cmd/gotosocial/action/admin/media/prune/common.go b/cmd/gotosocial/action/admin/media/prune/common.go index 07b9c13bb..bd759a0d4 100644 --- a/cmd/gotosocial/action/admin/media/prune/common.go +++ b/cmd/gotosocial/action/admin/media/prune/common.go @@ -38,21 +38,24 @@ type prune struct { func setupPrune(ctx context.Context) (*prune, error) { var state state.State state.Caches.Init() + state.Workers.Start() dbService, err := bundb.NewBunDBService(ctx, &state) if err != nil { return nil, fmt.Errorf("error creating dbservice: %w", err) } - storage, err := gtsstorage.AutoConfig() //nolint:contextcheck + //nolint:contextcheck + storage, err := gtsstorage.AutoConfig() if err != nil { return nil, fmt.Errorf("error creating storage backend: %w", err) } - manager, err := media.NewManager(dbService, storage) //nolint:contextcheck - if err != nil { - return nil, fmt.Errorf("error instantiating mediamanager: %w", err) - } + state.DB = dbService + state.Storage = storage + + //nolint:contextcheck + manager := media.NewManager(&state) return &prune{ dbService: dbService, @@ -70,9 +73,5 @@ func (p *prune) shutdown(ctx context.Context) error { return fmt.Errorf("error closing dbservice: %w", err) } - if err := p.manager.Stop(); err != nil { - return fmt.Errorf("error closing media manager: %w", err) - } - return nil } diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go index 74a5be24b..89486a5dc 100644 --- a/cmd/gotosocial/action/server/server.go +++ b/cmd/gotosocial/action/server/server.go @@ -91,33 +91,35 @@ var Start action.GTSAction = func(ctx context.Context) error { return fmt.Errorf("error creating instance instance: %s", err) } - // Create the client API and federator worker pools - // NOTE: these MUST NOT be used until they are passed to the - // processor and it is started. The reason being that the processor - // sets the Worker process functions and start the underlying pools - clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) - fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - - federatingDB := federatingdb.New(dbService, fedWorker) - - // build converters and util - typeConverter := typeutils.NewConverter(dbService) - // Open the storage backend storage, err := gtsstorage.AutoConfig() if err != nil { return fmt.Errorf("error creating storage backend: %w", err) } + // Set the state storage driver + state.Storage = storage + // Build HTTP client (TODO: add configurables here) client := httpclient.New(httpclient.Config{}) + // Initialize workers. + state.Workers.Start() + defer state.Workers.Stop() + + // Create the client API and federator worker pools + // NOTE: these MUST NOT be used until they are passed to the + // processor and it is started. The reason being that the processor + // sets the Worker process functions and start the underlying pools + // TODO: move these into state.Workers (and maybe reformat worker pools). + clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) + fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) + // build backend handlers - mediaManager, err := media.NewManager(dbService, storage) - if err != nil { - return fmt.Errorf("error creating media manager: %s", err) - } + mediaManager := media.NewManager(&state) oauthServer := oauth.New(ctx, dbService) + typeConverter := typeutils.NewConverter(dbService) + federatingDB := federatingdb.New(dbService, fedWorker, typeConverter) transportController := transport.NewController(dbService, federatingDB, &federation.Clock{}, client) federator := federation.NewFederator(dbService, federatingDB, transportController, typeConverter, mediaManager) diff --git a/cmd/gotosocial/main.go b/cmd/gotosocial/main.go index 30758b8c6..e3af77548 100644 --- a/cmd/gotosocial/main.go +++ b/cmd/gotosocial/main.go @@ -20,9 +20,10 @@ package main import ( "log" - "runtime/debug" + godebug "runtime/debug" "strings" + "codeberg.org/gruf/go-debug" "github.com/spf13/cobra" _ "github.com/superseriousbusiness/gotosocial/docs" @@ -60,9 +61,12 @@ func main() { // add subcommands rootCmd.AddCommand(serverCommands()) - rootCmd.AddCommand(testrigCommands()) rootCmd.AddCommand(debugCommands()) rootCmd.AddCommand(adminCommands()) + if debug.DEBUG { + // only add testrig if debug enabled. + rootCmd.AddCommand(testrigCommands()) + } // run if err := rootCmd.Execute(); err != nil { @@ -73,7 +77,7 @@ func main() { // version will build a version string from binary's stored build information. func version() string { // Read build information from binary - build, ok := debug.ReadBuildInfo() + build, ok := godebug.ReadBuildInfo() if !ok { return "" } @@ -6,13 +6,14 @@ require ( codeberg.org/gruf/go-bytesize v1.0.2 codeberg.org/gruf/go-byteutil v1.0.2 codeberg.org/gruf/go-cache/v3 v3.2.2 - codeberg.org/gruf/go-debug v1.2.0 + codeberg.org/gruf/go-debug v1.3.0 codeberg.org/gruf/go-errors/v2 v2.1.1 codeberg.org/gruf/go-fastcopy v1.1.2 codeberg.org/gruf/go-kv v1.5.2 codeberg.org/gruf/go-logger/v2 v2.2.1 codeberg.org/gruf/go-mutexes v1.1.5 - codeberg.org/gruf/go-runners v1.5.1 + codeberg.org/gruf/go-runners v1.6.0 + codeberg.org/gruf/go-sched v1.2.3 codeberg.org/gruf/go-store/v2 v2.2.1 github.com/KimMachineGun/automemlimit v0.2.4 github.com/abema/go-mp4 v0.10.0 @@ -37,7 +38,6 @@ require ( github.com/minio/minio-go/v7 v7.0.48 github.com/mitchellh/mapstructure v1.5.0 github.com/oklog/ulid v1.3.1 - github.com/robfig/cron/v3 v3.0.1 github.com/spf13/cobra v1.6.1 github.com/spf13/viper v1.15.0 github.com/stretchr/testify v1.8.1 @@ -66,7 +66,7 @@ require ( require ( codeberg.org/gruf/go-atomics v1.1.0 // indirect - codeberg.org/gruf/go-bitutil v1.0.1 // indirect + codeberg.org/gruf/go-bitutil v1.1.0 // indirect codeberg.org/gruf/go-bytes v1.0.2 // indirect codeberg.org/gruf/go-fastpath v1.0.3 // indirect codeberg.org/gruf/go-fastpath/v2 v2.0.0 // indirect @@ -75,7 +75,6 @@ require ( codeberg.org/gruf/go-mangler v1.2.2 // indirect codeberg.org/gruf/go-maps v1.0.3 // indirect codeberg.org/gruf/go-pools v1.1.0 // indirect - codeberg.org/gruf/go-sched v1.2.0 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/cilium/ebpf v0.4.0 // indirect github.com/containerd/cgroups v1.0.4 // indirect @@ -39,8 +39,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f codeberg.org/gruf/go-atomics v1.1.0 h1:ni9QXYoRUFYQMXE3akWaUb1wMcPBDc05Md6Rgml7W58= codeberg.org/gruf/go-atomics v1.1.0/go.mod h1:a/4/y/LgvjxjQVnpoy1VVkOSzLS1W9i1g4SJ0nflAa4= codeberg.org/gruf/go-bitutil v1.0.0/go.mod h1:sb8IjlDnjVTz8zPK/8lmHesKxY0Yb3iqHWjUM/SkphA= -codeberg.org/gruf/go-bitutil v1.0.1 h1:l8z9nOvCpHhicU2LZyJ6jLK03UNzCF6bxVCwu+VEenQ= -codeberg.org/gruf/go-bitutil v1.0.1/go.mod h1:3ezHnADoiRJs9jgn65AEZ3HY7dsabAYLmmnIvseCGJI= +codeberg.org/gruf/go-bitutil v1.1.0 h1:U1Q+A1mtnPk+npqYrlRBc9ar2C5hYiBd17l1Wrp2Bt8= +codeberg.org/gruf/go-bitutil v1.1.0/go.mod h1:rGibFevYTQfYKcPv0Df5KpG8n5xC3AfD4d/UgYeoNy0= codeberg.org/gruf/go-bytes v1.0.0/go.mod h1:1v/ibfaosfXSZtRdW2rWaVrDXMc9E3bsi/M9Ekx39cg= codeberg.org/gruf/go-bytes v1.0.2 h1:malqE42Ni+h1nnYWBUAJaDDtEzF4aeN4uPN8DfMNNvo= codeberg.org/gruf/go-bytes v1.0.2/go.mod h1:1v/ibfaosfXSZtRdW2rWaVrDXMc9E3bsi/M9Ekx39cg= @@ -51,8 +51,8 @@ codeberg.org/gruf/go-byteutil v1.0.2 h1:OesVyK5VKWeWdeDR00zRJ+Oy8hjXx1pBhn7WVvcZ codeberg.org/gruf/go-byteutil v1.0.2/go.mod h1:cWM3tgMCroSzqoBXUXMhvxTxYJp+TbCr6ioISRY5vSU= codeberg.org/gruf/go-cache/v3 v3.2.2 h1:hq6/RITgpcArjzbYSyo3uFxfIw7wW3KqAQjEaN7dj58= codeberg.org/gruf/go-cache/v3 v3.2.2/go.mod h1:+Eje6nCvN8QF71VyYjMWMnkdv6t1kHnCO/SvyC4K12Q= -codeberg.org/gruf/go-debug v1.2.0 h1:WBbTMnK1ArFKUmgv04aO2JiC/daTOB8zQGi521qb7OU= -codeberg.org/gruf/go-debug v1.2.0/go.mod h1:N+vSy9uJBQgpQcJUqjctvqFz7tBHJf+S/PIjLILzpLg= +codeberg.org/gruf/go-debug v1.3.0 h1:PIRxQiWUFKtGOGZFdZ3Y0pqyfI0Xr87j224IYe2snZs= +codeberg.org/gruf/go-debug v1.3.0/go.mod h1:N+vSy9uJBQgpQcJUqjctvqFz7tBHJf+S/PIjLILzpLg= codeberg.org/gruf/go-errors/v2 v2.0.0/go.mod h1:ZRhbdhvgoUA3Yw6e56kd9Ox984RrvbEFC2pOXyHDJP4= codeberg.org/gruf/go-errors/v2 v2.1.1 h1:oj7JUIvUBafF60HrwN74JrCMol1Ouh3gq1ggrH5hGTw= codeberg.org/gruf/go-errors/v2 v2.1.1/go.mod h1:LfzD9nkAAJpEDbkUqOZQ2jdaQ8VrK0pnR36zLOMFq6Y= @@ -79,10 +79,10 @@ codeberg.org/gruf/go-mutexes v1.1.5 h1:8Y8DwCGf24MyzOSaPvLrtk/B4ecVx4z+fppL6dY+P codeberg.org/gruf/go-mutexes v1.1.5/go.mod h1:1j/6/MBeBQUedAtAtysLLnBKogfOZAxdym0E3wlaBD8= codeberg.org/gruf/go-pools v1.1.0 h1:LbYP24eQLl/YI1fSU2pafiwhGol1Z1zPjRrMsXpF88s= codeberg.org/gruf/go-pools v1.1.0/go.mod h1:ZMYpt/DjQWYC3zFD3T97QWSFKs62zAUGJ/tzvgB9D68= -codeberg.org/gruf/go-runners v1.5.1 h1:ekhhxKvO6D/VC7nS/xpv71/iRX01JSqcBEbahqPUghg= -codeberg.org/gruf/go-runners v1.5.1/go.mod h1:kUM6GYL7dC+f9Sc/XuwdvB/mB4FuI4fJFb150ADMsmw= -codeberg.org/gruf/go-sched v1.2.0 h1:utZl/7srVcbh30rFw42LC2/cMtak4UZRxtIOt/5riNA= -codeberg.org/gruf/go-sched v1.2.0/go.mod h1:v4ueWq+fAtAw9JYt4aFXvadI1YoOqofgHQgszRYuslA= +codeberg.org/gruf/go-runners v1.6.0 h1:cAHKxMgtkb3v6it4qZZs4fo+yYgICNCrYvFlayuvSdk= +codeberg.org/gruf/go-runners v1.6.0/go.mod h1:QRcSExqXX8DM0rm8Xs6qX7baOzyvw0JIe4mu3TsQT+Y= +codeberg.org/gruf/go-sched v1.2.3 h1:H5ViDxxzOBR3uIyGBCf0eH8b1L8wMybOXcdtUUTXZHk= +codeberg.org/gruf/go-sched v1.2.3/go.mod h1:vT9uB6KWFIIwnG9vcPY2a0alYNoqdL1mSzRM8I+PK7A= codeberg.org/gruf/go-store/v2 v2.2.1 h1:lbvMjhMLebefiaPNLtWvPySKSYM5xN1aztSxxz+vCzU= codeberg.org/gruf/go-store/v2 v2.2.1/go.mod h1:pxdyfSzau8fFs1TfZlyRzhDYvZWLaj1sXpcjXpzBB6k= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -478,8 +478,6 @@ github.com/quasoft/memstore v0.0.0-20191010062613-2bce066d2b0b h1:aUNXCGgukb4gtY github.com/quasoft/memstore v0.0.0-20191010062613-2bce066d2b0b/go.mod h1:wTPjTepVu7uJBYgZ0SdWHQlIas582j6cn2jgk4DDdlg= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= -github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go index 82b69c7a1..143cafc9b 100644 --- a/internal/federation/dereferencing/account.go +++ b/internal/federation/dereferencing/account.go @@ -356,7 +356,7 @@ func (d *deref) fetchRemoteAccountAvatar(ctx context.Context, tsport transport.T } // Create new media processing request from the media manager instance. - processing, err := d.mediaManager.ProcessMedia(ctx, data, nil, accountID, &media.AdditionalMediaInfo{ + processing, err := d.mediaManager.PreProcessMedia(ctx, data, nil, accountID, &media.AdditionalMediaInfo{ Avatar: func() *bool { v := false; return &v }(), RemoteURL: &avatarURL, }) @@ -407,7 +407,7 @@ func (d *deref) fetchRemoteAccountHeader(ctx context.Context, tsport transport.T } // Create new media processing request from the media manager instance. - processing, err := d.mediaManager.ProcessMedia(ctx, data, nil, accountID, &media.AdditionalMediaInfo{ + processing, err := d.mediaManager.PreProcessMedia(ctx, data, nil, accountID, &media.AdditionalMediaInfo{ Header: func() *bool { v := true; return &v }(), RemoteURL: &headerURL, }) diff --git a/internal/federation/dereferencing/emoji.go b/internal/federation/dereferencing/emoji.go index 3e9452f14..7d8526c73 100644 --- a/internal/federation/dereferencing/emoji.go +++ b/internal/federation/dereferencing/emoji.go @@ -61,7 +61,7 @@ func (d *deref) GetRemoteEmoji(ctx context.Context, requestingUsername string, r return t.DereferenceMedia(innerCtx, derefURI) } - newProcessing, err := d.mediaManager.ProcessEmoji(ctx, dataFunc, nil, shortcode, id, emojiURI, ai, refresh) + newProcessing, err := d.mediaManager.PreProcessEmoji(ctx, dataFunc, nil, shortcode, id, emojiURI, ai, refresh) if err != nil { return nil, fmt.Errorf("GetRemoteEmoji: error processing emoji %s: %s", shortcodeDomain, err) } @@ -146,7 +146,6 @@ func (d *deref) populateEmojis(ctx context.Context, rawEmojis []*gtsmodel.Emoji, Disabled: gotEmoji.Disabled, VisibleInPicker: gotEmoji.VisibleInPicker, }, refresh) - if err != nil { log.Errorf("populateEmojis: couldn't refresh remote emoji %s: %s", shortcodeDomain, err) continue @@ -172,7 +171,6 @@ func (d *deref) populateEmojis(ctx context.Context, rawEmojis []*gtsmodel.Emoji, Disabled: e.Disabled, VisibleInPicker: e.VisibleInPicker, }, refresh) - if err != nil { log.Errorf("populateEmojis: couldn't get remote emoji %s: %s", shortcodeDomain, err) continue diff --git a/internal/federation/federatingdb/db.go b/internal/federation/federatingdb/db.go index 0787cc7cf..24455a553 100644 --- a/internal/federation/federatingdb/db.go +++ b/internal/federation/federatingdb/db.go @@ -49,12 +49,12 @@ type federatingDB struct { } // New returns a DB interface using the given database and config -func New(db db.DB, fedWorker *concurrency.WorkerPool[messages.FromFederator]) DB { +func New(db db.DB, fedWorker *concurrency.WorkerPool[messages.FromFederator], tc typeutils.TypeConverter) DB { fdb := federatingDB{ locks: mutexes.NewMap(-1, -1), // use defaults db: db, fedWorker: fedWorker, - typeConverter: typeutils.NewConverter(db), + typeConverter: tc, } return &fdb } diff --git a/internal/gotosocial/gotosocial.go b/internal/gotosocial/gotosocial.go index dfcab1a5e..e0a0a3b93 100644 --- a/internal/gotosocial/gotosocial.go +++ b/internal/gotosocial/gotosocial.go @@ -73,9 +73,6 @@ func (gts *gotosocial) Stop(ctx context.Context) error { if err := gts.apiRouter.Stop(ctx); err != nil { return err } - if err := gts.mediaManager.Stop(); err != nil { - return err - } if err := gts.db.Stop(ctx); err != nil { return err } diff --git a/internal/media/cron.go b/internal/media/cron.go deleted file mode 100644 index e32a63661..000000000 --- a/internal/media/cron.go +++ /dev/null @@ -1,73 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -package media - -import ( - "context" - "fmt" - "time" - - "github.com/robfig/cron/v3" - "github.com/superseriousbusiness/gotosocial/internal/config" - "github.com/superseriousbusiness/gotosocial/internal/log" -) - -type cronLogger struct{} - -func (l *cronLogger) Info(msg string, keysAndValues ...interface{}) { - log.Info("media manager cron logger: ", msg, keysAndValues) -} - -func (l *cronLogger) Error(err error, msg string, keysAndValues ...interface{}) { - log.Error("media manager cron logger: ", err, msg, keysAndValues) -} - -func scheduleCleanup(m *manager) error { - pruneCtx, pruneCancel := context.WithCancel(context.Background()) - - c := cron.New(cron.WithLogger(new(cronLogger))) - defer c.Start() - - if _, err := c.AddFunc("@midnight", func() { - if err := m.PruneAll(pruneCtx, config.GetMediaRemoteCacheDays(), true); err != nil { - log.Error(err) - return - } - }); err != nil { - pruneCancel() - return fmt.Errorf("error starting media manager cleanup job: %s", err) - } - - m.stopCronJobs = func() error { - // Try to stop jobs gracefully by waiting til they're finished. - stopCtx := c.Stop() - - select { - case <-stopCtx.Done(): - log.Infof("media manager: cron finished jobs and stopped gracefully") - case <-time.After(1 * time.Minute): - log.Warnf("media manager: cron didn't stop after 60 seconds, force closing jobs") - pruneCancel() - } - - return nil - } - - return nil -} diff --git a/internal/media/manager.go b/internal/media/manager.go index b770a7dcd..ba89aff13 100644 --- a/internal/media/manager.go +++ b/internal/media/manager.go @@ -20,11 +20,19 @@ package media import ( "context" + "errors" "fmt" + "time" - "github.com/superseriousbusiness/gotosocial/internal/concurrency" - "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/storage" + "codeberg.org/gruf/go-runners" + "codeberg.org/gruf/go-sched" + "codeberg.org/gruf/go-store/v2/storage" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/log" + "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/superseriousbusiness/gotosocial/internal/uris" ) var SupportedMIMETypes = []string{ @@ -42,16 +50,11 @@ var SupportedEmojiMIMETypes = []string{ // Manager provides an interface for managing media: parsing, storing, and retrieving media objects like photos, videos, and gifs. type Manager interface { - // Stop stops the underlying worker pool of the manager. It should be called - // when closing GoToSocial in order to cleanly finish any in-progress jobs. - // It will block until workers are finished processing. - Stop() error - /* PROCESSING FUNCTIONS */ - // ProcessMedia begins the process of decoding and storing the given data as an attachment. + // PreProcessMedia begins the process of decoding and storing the given data as an attachment. // It will return a pointer to a ProcessingMedia struct upon which further actions can be performed, such as getting // the finished media, thumbnail, attachment, etc. // @@ -63,8 +66,19 @@ type Manager interface { // accountID should be the account that the media belongs to. // // ai is optional and can be nil. Any additional information about the attachment provided will be put in the database. + // + // Note: unlike ProcessMedia, this will NOT queue the media to be asynchronously processed. + PreProcessMedia(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, accountID string, ai *AdditionalMediaInfo) (*ProcessingMedia, error) + + // PreProcessMediaRecache refetches, reprocesses, and recaches an existing attachment that has been uncached via pruneRemote. + // + // Note: unlike ProcessMedia, this will NOT queue the media to be asychronously processed. + PreProcessMediaRecache(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, attachmentID string) (*ProcessingMedia, error) + + // ProcessMedia will call PreProcessMedia, followed by queuing the media to be processing in the media worker queue. ProcessMedia(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, accountID string, ai *AdditionalMediaInfo) (*ProcessingMedia, error) - // ProcessEmoji begins the process of decoding and storing the given data as an emoji. + + // PreProcessEmoji begins the process of decoding and storing the given data as an emoji. // It will return a pointer to a ProcessingEmoji struct upon which further actions can be performed, such as getting // the finished media, thumbnail, attachment, etc. // @@ -81,10 +95,11 @@ type Manager interface { // // ai is optional and can be nil. Any additional information about the emoji provided will be put in the database. // - // If refresh is true, this indicates that the emoji image has changed and should be updated. + // Note: unlike ProcessEmoji, this will NOT queue the emoji to be asynchronously processed. + PreProcessEmoji(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo, refresh bool) (*ProcessingEmoji, error) + + // ProcessEmoji will call PreProcessEmoji, followed by queuing the emoji to be processing in the emoji worker queue. ProcessEmoji(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo, refresh bool) (*ProcessingEmoji, error) - // RecacheMedia refetches, reprocesses, and recaches an existing attachment that has been uncached via pruneRemote. - RecacheMedia(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, attachmentID string) (*ProcessingMedia, error) /* PRUNING/UNCACHING FUNCTIONS @@ -139,11 +154,7 @@ type Manager interface { } type manager struct { - db db.DB - storage *storage.Driver - emojiWorker *concurrency.WorkerPool[*ProcessingEmoji] - mediaWorker *concurrency.WorkerPool[*ProcessingMedia] - stopCronJobs func() error + state *state.State } // NewManager returns a media manager with the given db and underlying storage. @@ -152,88 +163,299 @@ type manager struct { // a limited number of media will be processed in parallel. The numbers of workers // is determined from the $GOMAXPROCS environment variable (usually no. CPU cores). // See internal/concurrency.NewWorkerPool() documentation for further information. -func NewManager(database db.DB, storage *storage.Driver) (Manager, error) { - m := &manager{ - db: database, - storage: storage, +func NewManager(state *state.State) Manager { + m := &manager{state: state} + scheduleCleanupJobs(m) + return m +} + +func (m *manager) PreProcessMedia(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, accountID string, ai *AdditionalMediaInfo) (*ProcessingMedia, error) { + id, err := id.NewRandomULID() + if err != nil { + return nil, err } - // Prepare the media worker pool. - m.mediaWorker = concurrency.NewWorkerPool[*ProcessingMedia](-1, 10) - m.mediaWorker.SetProcessor(func(ctx context.Context, media *ProcessingMedia) error { - if _, err := media.LoadAttachment(ctx); err != nil { - return fmt.Errorf("error loading media %s: %v", media.AttachmentID(), err) + avatar := false + header := false + cached := false + now := time.Now() + + // populate initial fields on the media attachment -- some of these will be overwritten as we proceed + attachment := >smodel.MediaAttachment{ + ID: id, + CreatedAt: now, + UpdatedAt: now, + StatusID: "", + URL: "", // we don't know yet because it depends on the uncalled DataFunc + RemoteURL: "", + Type: gtsmodel.FileTypeUnknown, // we don't know yet because it depends on the uncalled DataFunc + FileMeta: gtsmodel.FileMeta{}, + AccountID: accountID, + Description: "", + ScheduledStatusID: "", + Blurhash: "", + Processing: gtsmodel.ProcessingStatusReceived, + File: gtsmodel.File{UpdatedAt: now}, + Thumbnail: gtsmodel.Thumbnail{UpdatedAt: now}, + Avatar: &avatar, + Header: &header, + Cached: &cached, + } + + // check if we have additional info to add to the attachment, + // and overwrite some of the attachment fields if so + if ai != nil { + if ai.CreatedAt != nil { + attachment.CreatedAt = *ai.CreatedAt } - return nil - }) - // Prepare the emoji worker pool. - m.emojiWorker = concurrency.NewWorkerPool[*ProcessingEmoji](-1, 10) - m.emojiWorker.SetProcessor(func(ctx context.Context, emoji *ProcessingEmoji) error { - if _, err := emoji.LoadEmoji(ctx); err != nil { - return fmt.Errorf("error loading emoji %s: %v", emoji.EmojiID(), err) + if ai.StatusID != nil { + attachment.StatusID = *ai.StatusID } - return nil - }) - // Start the worker pools. - if err := m.mediaWorker.Start(); err != nil { - return nil, err + if ai.RemoteURL != nil { + attachment.RemoteURL = *ai.RemoteURL + } + + if ai.Description != nil { + attachment.Description = *ai.Description + } + + if ai.ScheduledStatusID != nil { + attachment.ScheduledStatusID = *ai.ScheduledStatusID + } + + if ai.Blurhash != nil { + attachment.Blurhash = *ai.Blurhash + } + + if ai.Avatar != nil { + attachment.Avatar = ai.Avatar + } + + if ai.Header != nil { + attachment.Header = ai.Header + } + + if ai.FocusX != nil { + attachment.FileMeta.Focus.X = *ai.FocusX + } + + if ai.FocusY != nil { + attachment.FileMeta.Focus.Y = *ai.FocusY + } } - if err := m.emojiWorker.Start(); err != nil { - return nil, err + + processingMedia := &ProcessingMedia{ + media: attachment, + dataFn: data, + postFn: postData, + mgr: m, } - // Schedule cron job(s) for clean up. - if err := scheduleCleanup(m); err != nil { + return processingMedia, nil +} + +func (m *manager) PreProcessMediaRecache(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, attachmentID string) (*ProcessingMedia, error) { + // get the existing attachment from database. + attachment, err := m.state.DB.GetAttachmentByID(ctx, attachmentID) + if err != nil { return nil, err } - return m, nil + processingMedia := &ProcessingMedia{ + media: attachment, + dataFn: data, + postFn: postData, + recache: true, // indicate it's a recache + mgr: m, + } + + return processingMedia, nil } func (m *manager) ProcessMedia(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, accountID string, ai *AdditionalMediaInfo) (*ProcessingMedia, error) { - processingMedia, err := m.preProcessMedia(ctx, data, postData, accountID, ai) + // Create a new processing media object for this media request. + media, err := m.PreProcessMedia(ctx, data, postData, accountID, ai) if err != nil { return nil, err } - m.mediaWorker.Queue(processingMedia) - return processingMedia, nil + + // Attempt to add this media processing item to the worker queue. + _ = m.state.Workers.Media.MustEnqueueCtx(ctx, media.Process) + + return media, nil } -func (m *manager) ProcessEmoji(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo, refresh bool) (*ProcessingEmoji, error) { - processingEmoji, err := m.preProcessEmoji(ctx, data, postData, shortcode, id, uri, ai, refresh) +func (m *manager) PreProcessEmoji(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, shortcode string, emojiID string, uri string, ai *AdditionalEmojiInfo, refresh bool) (*ProcessingEmoji, error) { + instanceAccount, err := m.state.DB.GetInstanceAccount(ctx, "") if err != nil { - return nil, err + return nil, fmt.Errorf("preProcessEmoji: error fetching this instance account from the db: %s", err) + } + + var ( + newPathID string + emoji *gtsmodel.Emoji + now = time.Now() + ) + + if refresh { + emoji, err = m.state.DB.GetEmojiByID(ctx, emojiID) + if err != nil { + return nil, fmt.Errorf("preProcessEmoji: error fetching emoji to refresh from the db: %s", err) + } + + // if this is a refresh, we will end up with new images + // stored for this emoji, so we can use the postData function + // to perform clean up of the old images from storage + originalPostData := postData + originalImagePath := emoji.ImagePath + originalImageStaticPath := emoji.ImageStaticPath + postData = func(innerCtx context.Context) error { + // trigger the original postData function if it was provided + if originalPostData != nil { + if err := originalPostData(innerCtx); err != nil { + return err + } + } + + l := log.WithField("shortcode@domain", emoji.Shortcode+"@"+emoji.Domain) + l.Debug("postData: cleaning up old emoji files for refreshed emoji") + if err := m.state.Storage.Delete(innerCtx, originalImagePath); err != nil && !errors.Is(err, storage.ErrNotFound) { + l.Errorf("postData: error cleaning up old emoji image at %s for refreshed emoji: %s", originalImagePath, err) + } + if err := m.state.Storage.Delete(innerCtx, originalImageStaticPath); err != nil && !errors.Is(err, storage.ErrNotFound) { + l.Errorf("postData: error cleaning up old emoji static image at %s for refreshed emoji: %s", originalImageStaticPath, err) + } + + return nil + } + + newPathID, err = id.NewRandomULID() + if err != nil { + return nil, fmt.Errorf("preProcessEmoji: error generating alternateID for emoji refresh: %s", err) + } + + // store + serve static image at new path ID + emoji.ImageStaticURL = uris.GenerateURIForAttachment(instanceAccount.ID, string(TypeEmoji), string(SizeStatic), newPathID, mimePng) + emoji.ImageStaticPath = fmt.Sprintf("%s/%s/%s/%s.%s", instanceAccount.ID, TypeEmoji, SizeStatic, newPathID, mimePng) + + emoji.Shortcode = shortcode + emoji.URI = uri + } else { + disabled := false + visibleInPicker := true + + // populate initial fields on the emoji -- some of these will be overwritten as we proceed + emoji = >smodel.Emoji{ + ID: emojiID, + CreatedAt: now, + Shortcode: shortcode, + Domain: "", // assume our own domain unless told otherwise + ImageRemoteURL: "", + ImageStaticRemoteURL: "", + ImageURL: "", // we don't know yet + ImageStaticURL: uris.GenerateURIForAttachment(instanceAccount.ID, string(TypeEmoji), string(SizeStatic), emojiID, mimePng), // all static emojis are encoded as png + ImagePath: "", // we don't know yet + ImageStaticPath: fmt.Sprintf("%s/%s/%s/%s.%s", instanceAccount.ID, TypeEmoji, SizeStatic, emojiID, mimePng), // all static emojis are encoded as png + ImageContentType: "", // we don't know yet + ImageStaticContentType: mimeImagePng, // all static emojis are encoded as png + ImageFileSize: 0, + ImageStaticFileSize: 0, + Disabled: &disabled, + URI: uri, + VisibleInPicker: &visibleInPicker, + CategoryID: "", + } + } + + emoji.ImageUpdatedAt = now + emoji.UpdatedAt = now + + // check if we have additional info to add to the emoji, + // and overwrite some of the emoji fields if so + if ai != nil { + if ai.CreatedAt != nil { + emoji.CreatedAt = *ai.CreatedAt + } + + if ai.Domain != nil { + emoji.Domain = *ai.Domain + } + + if ai.ImageRemoteURL != nil { + emoji.ImageRemoteURL = *ai.ImageRemoteURL + } + + if ai.ImageStaticRemoteURL != nil { + emoji.ImageStaticRemoteURL = *ai.ImageStaticRemoteURL + } + + if ai.Disabled != nil { + emoji.Disabled = ai.Disabled + } + + if ai.VisibleInPicker != nil { + emoji.VisibleInPicker = ai.VisibleInPicker + } + + if ai.CategoryID != nil { + emoji.CategoryID = *ai.CategoryID + } } - m.emojiWorker.Queue(processingEmoji) + + processingEmoji := &ProcessingEmoji{ + instAccID: instanceAccount.ID, + emoji: emoji, + refresh: refresh, + newPathID: newPathID, + dataFn: data, + postFn: postData, + mgr: m, + } + return processingEmoji, nil } -func (m *manager) RecacheMedia(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, attachmentID string) (*ProcessingMedia, error) { - processingRecache, err := m.preProcessRecache(ctx, data, postData, attachmentID) +func (m *manager) ProcessEmoji(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo, refresh bool) (*ProcessingEmoji, error) { + // Create a new processing emoji object for this emoji request. + emoji, err := m.PreProcessEmoji(ctx, data, postData, shortcode, id, uri, ai, refresh) if err != nil { return nil, err } - m.mediaWorker.Queue(processingRecache) - return processingRecache, nil + + // Attempt to add this emoji processing item to the worker queue. + _ = m.state.Workers.Media.MustEnqueueCtx(ctx, emoji.Process) + + return emoji, nil } -func (m *manager) Stop() error { - // Stop worker pools. - mediaErr := m.mediaWorker.Stop() - emojiErr := m.emojiWorker.Stop() +func scheduleCleanupJobs(m *manager) { + const day = time.Hour * 24 - var cronErr error - if m.stopCronJobs != nil { - cronErr = m.stopCronJobs() - } + // Calculate closest midnight. + now := time.Now() + midnight := now.Round(day) - if mediaErr != nil { - return mediaErr - } else if emojiErr != nil { - return emojiErr + if midnight.Before(now) { + // since <= 11:59am rounds down. + midnight = midnight.Add(day) } - return cronErr + // Get ctx associated with scheduler run state. + done := m.state.Workers.Scheduler.Done() + doneCtx := runners.CancelCtx(done) + + // TODO: we'll need to do some thinking to make these + // jobs restartable if we want to implement reloads in + // the future that make call to Workers.Stop() -> Workers.Start(). + + // Schedule the PruneAll task to execute every day at midnight. + m.state.Workers.Scheduler.Schedule(sched.NewJob(func(now time.Time) { + err := m.PruneAll(doneCtx, config.GetMediaRemoteCacheDays(), true) + if err != nil { + log.Errorf("error during prune: %v", err) + } + log.Infof("finished pruning all in %s", time.Since(now)) + }).EveryAt(midnight, day)) } diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go index d912c9d87..f4dd8dac7 100644 --- a/internal/media/manager_test.go +++ b/internal/media/manager_test.go @@ -33,6 +33,7 @@ import ( "github.com/stretchr/testify/suite" gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/state" gtsstorage "github.com/superseriousbusiness/gotosocial/internal/storage" ) @@ -1189,15 +1190,19 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithDiskStorage() { panic(err) } + var state state.State + + state.Workers.Start() + defer state.Workers.Stop() + storage := >sstorage.Driver{ KVStore: kv.New(disk), Storage: disk, } + state.Storage = storage + state.DB = suite.db - diskManager, err := media.NewManager(suite.db, storage) - if err != nil { - panic(err) - } + diskManager := media.NewManager(&state) suite.manager = diskManager // process the media with no additional info provided diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go index b68c9dfe1..0a36174b0 100644 --- a/internal/media/processingemoji.go +++ b/internal/media/processingemoji.go @@ -21,18 +21,15 @@ package media import ( "bytes" "context" - "errors" "fmt" "io" - "sync" - "time" "codeberg.org/gruf/go-bytesize" - gostore "codeberg.org/gruf/go-store/v2/storage" + "codeberg.org/gruf/go-errors/v2" + "codeberg.org/gruf/go-runners" "github.com/h2non/filetype" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/uris" ) @@ -46,9 +43,10 @@ type ProcessingEmoji struct { newPathID string // new emoji path ID to use if refreshed dataFn DataFunc // load-data function, returns media stream postFn PostDataCallbackFunc // post data callback function - err error // error encountered during processing - manager *manager // manager instance (access to db / storage) - once sync.Once // once ensures processing only occurs once + done bool // done is set when process finishes with non ctx canceled type error + proc runners.Processor // proc helps synchronize only a singular running processing instance + err error // error stores permanent error value when done + mgr *manager // mgr instance (access to db / storage) } // EmojiID returns the ID of the underlying emoji without blocking processing. @@ -56,40 +54,72 @@ func (p *ProcessingEmoji) EmojiID() string { return p.emoji.ID // immutable, safe outside mutex. } -// LoadEmoji blocks until the static and fullsize image -// has been processed, and then returns the completed emoji. +// LoadEmoji blocks until the static and fullsize image has been processed, and then returns the completed emoji. func (p *ProcessingEmoji) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error) { - // only process once. - p.once.Do(func() { - var err error + // Attempt to load synchronously. + emoji, done, err := p.load(ctx) + + if err == nil { + // No issue, return media. + return emoji, nil + } + + if !done { + // Provided context was cancelled, e.g. request cancelled + // early. Queue this item for asynchronous processing. + log.Warnf("reprocessing emoji %s after canceled ctx", p.emoji.ID) + go p.mgr.state.Workers.Media.Enqueue(p.Process) + } + + return nil, err +} + +// Process allows the receiving object to fit the runners.WorkerFunc signature. It performs a (blocking) load and logs on error. +func (p *ProcessingEmoji) Process(ctx context.Context) { + if _, _, err := p.load(ctx); err != nil { + log.Errorf("error processing emoji: %v", err) + } +} + +// load performs a concurrency-safe load of ProcessingEmoji, only marking itself as complete when returned error is NOT a context cancel. +func (p *ProcessingEmoji) load(ctx context.Context) (*gtsmodel.Emoji, bool, error) { + var ( + done bool + err error + ) + + err = p.proc.Process(func() error { + if p.done { + // Already proc'd. + return p.err + } defer func() { - if r := recover(); r != nil { - if err != nil { - rOld := r // wrap the panic so we don't lose existing returned error - r = fmt.Errorf("panic occured after error %q: %v", err.Error(), rOld) - } - - // Catch any panics and wrap as error. - err = fmt.Errorf("caught panic: %v", r) + // This is only done when ctx NOT cancelled. + done = err == nil || !errors.Is(err, + context.Canceled, + context.DeadlineExceeded, + ) + + if !done { + return } - if err != nil { - // Store error. - p.err = err - } + // Store final values. + p.done = true + p.err = err }() // Attempt to store media and calculate // full-size media attachment details. if err = p.store(ctx); err != nil { - return + return err } // Finish processing by reloading media into // memory to get dimension and generate a thumb. if err = p.finish(ctx); err != nil { - return + return err } if p.refresh { @@ -110,20 +140,20 @@ func (p *ProcessingEmoji) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error } // Existing emoji we're refreshing, so only need to update. - _, err = p.manager.db.UpdateEmoji(ctx, p.emoji, columns...) - return + _, err = p.mgr.state.DB.UpdateEmoji(ctx, p.emoji, columns...) + return err } // New emoji media, first time caching. - err = p.manager.db.PutEmoji(ctx, p.emoji) - return //nolint shutup linter i like this here + err = p.mgr.state.DB.PutEmoji(ctx, p.emoji) + return err }) - if p.err != nil { - return nil, p.err + if err != nil { + return nil, done, err } - return p.emoji, nil + return p.emoji, done, nil } // store calls the data function attached to p if it hasn't been called yet, @@ -220,24 +250,24 @@ func (p *ProcessingEmoji) store(ctx context.Context) error { ) // This shouldn't already exist, but we do a check as it's worth logging. - if have, _ := p.manager.storage.Has(ctx, p.emoji.ImagePath); have { + if have, _ := p.mgr.state.Storage.Has(ctx, p.emoji.ImagePath); have { log.Warnf("emoji already exists at storage path: %s", p.emoji.ImagePath) // Attempt to remove existing emoji at storage path (might be broken / out-of-date) - if err := p.manager.storage.Delete(ctx, p.emoji.ImagePath); err != nil { + if err := p.mgr.state.Storage.Delete(ctx, p.emoji.ImagePath); err != nil { return fmt.Errorf("error removing emoji from storage: %v", err) } } // Write the final image reader stream to our storage. - sz, err = p.manager.storage.PutStream(ctx, p.emoji.ImagePath, r) + sz, err = p.mgr.state.Storage.PutStream(ctx, p.emoji.ImagePath, r) if err != nil { return fmt.Errorf("error writing emoji to storage: %w", err) } // Once again check size in case none was provided previously. if size := bytesize.Size(sz); size > maxSize { - if err := p.manager.storage.Delete(ctx, p.emoji.ImagePath); err != nil { + if err := p.mgr.state.Storage.Delete(ctx, p.emoji.ImagePath); err != nil { log.Errorf("error removing too-large-emoji from storage: %v", err) } return fmt.Errorf("calculated emoji size %s greater than max allowed %s", size, maxSize) @@ -259,7 +289,7 @@ func (p *ProcessingEmoji) store(ctx context.Context) error { func (p *ProcessingEmoji) finish(ctx context.Context) error { // Fetch a stream to the original file in storage. - rc, err := p.manager.storage.GetStream(ctx, p.emoji.ImagePath) + rc, err := p.mgr.state.Storage.GetStream(ctx, p.emoji.ImagePath) if err != nil { return fmt.Errorf("error loading file from storage: %w", err) } @@ -277,11 +307,11 @@ func (p *ProcessingEmoji) finish(ctx context.Context) error { } // This shouldn't already exist, but we do a check as it's worth logging. - if have, _ := p.manager.storage.Has(ctx, p.emoji.ImageStaticPath); have { + if have, _ := p.mgr.state.Storage.Has(ctx, p.emoji.ImageStaticPath); have { log.Warnf("static emoji already exists at storage path: %s", p.emoji.ImagePath) // Attempt to remove static existing emoji at storage path (might be broken / out-of-date) - if err := p.manager.storage.Delete(ctx, p.emoji.ImageStaticPath); err != nil { + if err := p.mgr.state.Storage.Delete(ctx, p.emoji.ImageStaticPath); err != nil { return fmt.Errorf("error removing static emoji from storage: %v", err) } } @@ -290,7 +320,7 @@ func (p *ProcessingEmoji) finish(ctx context.Context) error { enc := staticImg.ToPNG() // Stream-encode the PNG static image into storage. - sz, err := p.manager.storage.PutStream(ctx, p.emoji.ImageStaticPath, enc) + sz, err := p.mgr.state.Storage.PutStream(ctx, p.emoji.ImageStaticPath, enc) if err != nil { return fmt.Errorf("error stream-encoding static emoji to storage: %w", err) } @@ -300,129 +330,3 @@ func (p *ProcessingEmoji) finish(ctx context.Context) error { return nil } - -func (m *manager) preProcessEmoji(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, shortcode string, emojiID string, uri string, ai *AdditionalEmojiInfo, refresh bool) (*ProcessingEmoji, error) { - instanceAccount, err := m.db.GetInstanceAccount(ctx, "") - if err != nil { - return nil, fmt.Errorf("preProcessEmoji: error fetching this instance account from the db: %s", err) - } - - var newPathID string - var emoji *gtsmodel.Emoji - if refresh { - emoji, err = m.db.GetEmojiByID(ctx, emojiID) - if err != nil { - return nil, fmt.Errorf("preProcessEmoji: error fetching emoji to refresh from the db: %s", err) - } - - // if this is a refresh, we will end up with new images - // stored for this emoji, so we can use the postData function - // to perform clean up of the old images from storage - originalPostData := postData - originalImagePath := emoji.ImagePath - originalImageStaticPath := emoji.ImageStaticPath - postData = func(innerCtx context.Context) error { - // trigger the original postData function if it was provided - if originalPostData != nil { - if err := originalPostData(innerCtx); err != nil { - return err - } - } - - l := log.WithField("shortcode@domain", emoji.Shortcode+"@"+emoji.Domain) - l.Debug("postData: cleaning up old emoji files for refreshed emoji") - if err := m.storage.Delete(innerCtx, originalImagePath); err != nil && !errors.Is(err, gostore.ErrNotFound) { - l.Errorf("postData: error cleaning up old emoji image at %s for refreshed emoji: %s", originalImagePath, err) - } - if err := m.storage.Delete(innerCtx, originalImageStaticPath); err != nil && !errors.Is(err, gostore.ErrNotFound) { - l.Errorf("postData: error cleaning up old emoji static image at %s for refreshed emoji: %s", originalImageStaticPath, err) - } - - return nil - } - - newPathID, err = id.NewRandomULID() - if err != nil { - return nil, fmt.Errorf("preProcessEmoji: error generating alternateID for emoji refresh: %s", err) - } - - // store + serve static image at new path ID - emoji.ImageStaticURL = uris.GenerateURIForAttachment(instanceAccount.ID, string(TypeEmoji), string(SizeStatic), newPathID, mimePng) - emoji.ImageStaticPath = fmt.Sprintf("%s/%s/%s/%s.%s", instanceAccount.ID, TypeEmoji, SizeStatic, newPathID, mimePng) - - emoji.Shortcode = shortcode - emoji.URI = uri - } else { - disabled := false - visibleInPicker := true - - // populate initial fields on the emoji -- some of these will be overwritten as we proceed - emoji = >smodel.Emoji{ - ID: emojiID, - CreatedAt: time.Now(), - Shortcode: shortcode, - Domain: "", // assume our own domain unless told otherwise - ImageRemoteURL: "", - ImageStaticRemoteURL: "", - ImageURL: "", // we don't know yet - ImageStaticURL: uris.GenerateURIForAttachment(instanceAccount.ID, string(TypeEmoji), string(SizeStatic), emojiID, mimePng), // all static emojis are encoded as png - ImagePath: "", // we don't know yet - ImageStaticPath: fmt.Sprintf("%s/%s/%s/%s.%s", instanceAccount.ID, TypeEmoji, SizeStatic, emojiID, mimePng), // all static emojis are encoded as png - ImageContentType: "", // we don't know yet - ImageStaticContentType: mimeImagePng, // all static emojis are encoded as png - ImageFileSize: 0, - ImageStaticFileSize: 0, - Disabled: &disabled, - URI: uri, - VisibleInPicker: &visibleInPicker, - CategoryID: "", - } - } - - emoji.ImageUpdatedAt = time.Now() - emoji.UpdatedAt = time.Now() - - // check if we have additional info to add to the emoji, - // and overwrite some of the emoji fields if so - if ai != nil { - if ai.CreatedAt != nil { - emoji.CreatedAt = *ai.CreatedAt - } - - if ai.Domain != nil { - emoji.Domain = *ai.Domain - } - - if ai.ImageRemoteURL != nil { - emoji.ImageRemoteURL = *ai.ImageRemoteURL - } - - if ai.ImageStaticRemoteURL != nil { - emoji.ImageStaticRemoteURL = *ai.ImageStaticRemoteURL - } - - if ai.Disabled != nil { - emoji.Disabled = ai.Disabled - } - - if ai.VisibleInPicker != nil { - emoji.VisibleInPicker = ai.VisibleInPicker - } - - if ai.CategoryID != nil { - emoji.CategoryID = *ai.CategoryID - } - } - - processingEmoji := &ProcessingEmoji{ - instAccID: instanceAccount.ID, - emoji: emoji, - refresh: refresh, - newPathID: newPathID, - dataFn: data, - postFn: postData, - manager: m, - } - - return processingEmoji, nil -} diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go index 34f8dc26b..b4eda4072 100644 --- a/internal/media/processingmedia.go +++ b/internal/media/processingmedia.go @@ -24,14 +24,14 @@ import ( "fmt" "image/jpeg" "io" - "sync" "time" + "codeberg.org/gruf/go-errors/v2" + "codeberg.org/gruf/go-runners" "github.com/disintegration/imaging" "github.com/h2non/filetype" terminator "github.com/superseriousbusiness/exif-terminator" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/uris" ) @@ -40,12 +40,13 @@ import ( // various functions for retrieving data from the process. type ProcessingMedia struct { media *gtsmodel.MediaAttachment // processing media attachment details - recache bool // recaching existing (uncached) media dataFn DataFunc // load-data function, returns media stream postFn PostDataCallbackFunc // post data callback function - err error // error encountered during processing - manager *manager // manager instance (access to db / storage) - once sync.Once // once ensures processing only occurs once + recache bool // recaching existing (uncached) media + done bool // done is set when process finishes with non ctx canceled type error + proc runners.Processor // proc helps synchronize only a singular running processing instance + err error // error stores permanent error value when done + mgr *manager // mgr instance (access to db / storage) } // AttachmentID returns the ID of the underlying media attachment without blocking processing. @@ -53,58 +54,90 @@ func (p *ProcessingMedia) AttachmentID() string { return p.media.ID // immutable, safe outside mutex. } -// LoadAttachment blocks until the thumbnail and fullsize content -// has been processed, and then returns the completed attachment. +// LoadAttachment blocks until the thumbnail and fullsize content has been processed, and then returns the completed attachment. func (p *ProcessingMedia) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAttachment, error) { - // only process once. - p.once.Do(func() { - var err error + // Attempt to load synchronously. + media, done, err := p.load(ctx) + + if err == nil { + // No issue, return media. + return media, nil + } + + if !done { + // Provided context was cancelled, e.g. request cancelled + // early. Queue this item for asynchronous processing. + log.Warnf("reprocessing media %s after canceled ctx", p.media.ID) + go p.mgr.state.Workers.Media.Enqueue(p.Process) + } + + return nil, err +} + +// Process allows the receiving object to fit the runners.WorkerFunc signature. It performs a (blocking) load and logs on error. +func (p *ProcessingMedia) Process(ctx context.Context) { + if _, _, err := p.load(ctx); err != nil { + log.Errorf("error processing media: %v", err) + } +} + +// load performs a concurrency-safe load of ProcessingMedia, only marking itself as complete when returned error is NOT a context cancel. +func (p *ProcessingMedia) load(ctx context.Context) (*gtsmodel.MediaAttachment, bool, error) { + var ( + done bool + err error + ) + + err = p.proc.Process(func() error { + if p.done { + // Already proc'd. + return p.err + } defer func() { - if r := recover(); r != nil { - if err != nil { - rOld := r // wrap the panic so we don't lose existing returned error - r = fmt.Errorf("panic occured after error %q: %v", err.Error(), rOld) - } - - // Catch any panics and wrap as error. - err = fmt.Errorf("caught panic: %v", r) + // This is only done when ctx NOT cancelled. + done = err == nil || !errors.Is(err, + context.Canceled, + context.DeadlineExceeded, + ) + + if !done { + return } - if err != nil { - // Store error. - p.err = err - } + // Store final values. + p.done = true + p.err = err }() // Attempt to store media and calculate // full-size media attachment details. if err = p.store(ctx); err != nil { - return + return err } // Finish processing by reloading media into // memory to get dimension and generate a thumb. if err = p.finish(ctx); err != nil { - return + return err } if p.recache { // Existing attachment we're recaching, so only need to update. - err = p.manager.db.UpdateByID(ctx, p.media, p.media.ID) - return + err = p.mgr.state.DB.UpdateByID(ctx, p.media, p.media.ID) + return err } // New attachment, first time caching. - err = p.manager.db.Put(ctx, p.media) - return //nolint shutup linter i like this here + err = p.mgr.state.DB.Put(ctx, p.media) + return err }) - if p.err != nil { - return nil, p.err + if err != nil { + return nil, done, err } - return p.media, nil + return p.media, done, nil } // store calls the data function attached to p if it hasn't been called yet, @@ -186,17 +219,17 @@ func (p *ProcessingMedia) store(ctx context.Context) error { ) // This shouldn't already exist, but we do a check as it's worth logging. - if have, _ := p.manager.storage.Has(ctx, p.media.File.Path); have { + if have, _ := p.mgr.state.Storage.Has(ctx, p.media.File.Path); have { log.Warnf("media already exists at storage path: %s", p.media.File.Path) // Attempt to remove existing media at storage path (might be broken / out-of-date) - if err := p.manager.storage.Delete(ctx, p.media.File.Path); err != nil { + if err := p.mgr.state.Storage.Delete(ctx, p.media.File.Path); err != nil { return fmt.Errorf("error removing media from storage: %v", err) } } // Write the final image reader stream to our storage. - sz, err = p.manager.storage.PutStream(ctx, p.media.File.Path, r) + sz, err = p.mgr.state.Storage.PutStream(ctx, p.media.File.Path, r) if err != nil { return fmt.Errorf("error writing media to storage: %w", err) } @@ -221,7 +254,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error { func (p *ProcessingMedia) finish(ctx context.Context) error { // Fetch a stream to the original file in storage. - rc, err := p.manager.storage.GetStream(ctx, p.media.File.Path) + rc, err := p.mgr.state.Storage.GetStream(ctx, p.media.File.Path) if err != nil { return fmt.Errorf("error loading file from storage: %w", err) } @@ -299,11 +332,11 @@ func (p *ProcessingMedia) finish(ctx context.Context) error { p.media.Blurhash = hash // This shouldn't already exist, but we do a check as it's worth logging. - if have, _ := p.manager.storage.Has(ctx, p.media.Thumbnail.Path); have { + if have, _ := p.mgr.state.Storage.Has(ctx, p.media.Thumbnail.Path); have { log.Warnf("thumbnail already exists at storage path: %s", p.media.Thumbnail.Path) // Attempt to remove existing thumbnail at storage path (might be broken / out-of-date) - if err := p.manager.storage.Delete(ctx, p.media.Thumbnail.Path); err != nil { + if err := p.mgr.state.Storage.Delete(ctx, p.media.Thumbnail.Path); err != nil { return fmt.Errorf("error removing thumbnail from storage: %v", err) } } @@ -314,7 +347,7 @@ func (p *ProcessingMedia) finish(ctx context.Context) error { }) // Stream-encode the JPEG thumbnail image into storage. - sz, err := p.manager.storage.PutStream(ctx, p.media.Thumbnail.Path, enc) + sz, err := p.mgr.state.Storage.PutStream(ctx, p.media.Thumbnail.Path, enc) if err != nil { return fmt.Errorf("error stream-encoding thumbnail to storage: %w", err) } @@ -346,107 +379,3 @@ func (p *ProcessingMedia) finish(ctx context.Context) error { return nil } - -func (m *manager) preProcessMedia(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, accountID string, ai *AdditionalMediaInfo) (*ProcessingMedia, error) { - id, err := id.NewRandomULID() - if err != nil { - return nil, err - } - - avatar := false - header := false - cached := false - - // populate initial fields on the media attachment -- some of these will be overwritten as we proceed - attachment := >smodel.MediaAttachment{ - ID: id, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - StatusID: "", - URL: "", // we don't know yet because it depends on the uncalled DataFunc - RemoteURL: "", - Type: gtsmodel.FileTypeUnknown, // we don't know yet because it depends on the uncalled DataFunc - FileMeta: gtsmodel.FileMeta{}, - AccountID: accountID, - Description: "", - ScheduledStatusID: "", - Blurhash: "", - Processing: gtsmodel.ProcessingStatusReceived, - File: gtsmodel.File{UpdatedAt: time.Now()}, - Thumbnail: gtsmodel.Thumbnail{UpdatedAt: time.Now()}, - Avatar: &avatar, - Header: &header, - Cached: &cached, - } - - // check if we have additional info to add to the attachment, - // and overwrite some of the attachment fields if so - if ai != nil { - if ai.CreatedAt != nil { - attachment.CreatedAt = *ai.CreatedAt - } - - if ai.StatusID != nil { - attachment.StatusID = *ai.StatusID - } - - if ai.RemoteURL != nil { - attachment.RemoteURL = *ai.RemoteURL - } - - if ai.Description != nil { - attachment.Description = *ai.Description - } - - if ai.ScheduledStatusID != nil { - attachment.ScheduledStatusID = *ai.ScheduledStatusID - } - - if ai.Blurhash != nil { - attachment.Blurhash = *ai.Blurhash - } - - if ai.Avatar != nil { - attachment.Avatar = ai.Avatar - } - - if ai.Header != nil { - attachment.Header = ai.Header - } - - if ai.FocusX != nil { - attachment.FileMeta.Focus.X = *ai.FocusX - } - - if ai.FocusY != nil { - attachment.FileMeta.Focus.Y = *ai.FocusY - } - } - - processingMedia := &ProcessingMedia{ - media: attachment, - dataFn: data, - postFn: postData, - manager: m, - } - - return processingMedia, nil -} - -func (m *manager) preProcessRecache(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, id string) (*ProcessingMedia, error) { - // get the existing attachment from database. - attachment, err := m.db.GetAttachmentByID(ctx, id) - if err != nil { - return nil, err - } - - processingMedia := &ProcessingMedia{ - media: attachment, - dataFn: data, - postFn: postData, - manager: m, - recache: true, // indicate it's a recache - } - - return processingMedia, nil -} diff --git a/internal/media/prune.go b/internal/media/prune.go index 7335feb7a..3509e249d 100644 --- a/internal/media/prune.go +++ b/internal/media/prune.go @@ -72,7 +72,7 @@ func (m *manager) PruneAll(ctx context.Context, mediaCacheRemoteDays int, blocki log.Infof("pruned %d orphaned media", pruned) } - if err := m.storage.Storage.Clean(innerCtx); err != nil { + if err := m.state.Storage.Storage.Clean(innerCtx); err != nil { errs = append(errs, fmt.Sprintf("error cleaning storage: (%s)", err)) } else { log.Info("cleaned storage") @@ -116,7 +116,7 @@ func (m *manager) PruneUnusedRemote(ctx context.Context, dry bool) (int, error) } } - for attachments, err = m.db.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.db.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit) { + for attachments, err = m.state.DB.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.state.DB.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit) { maxID = attachments[len(attachments)-1].ID // use the id of the last attachment in the slice as the next 'maxID' value // Prune each attachment that meets one of the following criteria: @@ -157,7 +157,7 @@ func (m *manager) PruneOrphaned(ctx context.Context, dry bool) (int, error) { return false } - iterator, err := m.storage.Iterator(ctx, match) // make sure this iterator is always released + iterator, err := m.state.Storage.Iterator(ctx, match) // make sure this iterator is always released if err != nil { return 0, fmt.Errorf("PruneOrphaned: error getting storage iterator: %w", err) } @@ -172,7 +172,7 @@ func (m *manager) PruneOrphaned(ctx context.Context, dry bool) (int, error) { // Emojis are stored under the instance account, // so we need the ID of the instance account for // the next part. - instanceAccount, err := m.db.GetInstanceAccount(ctx, "") + instanceAccount, err := m.state.DB.GetInstanceAccount(ctx, "") if err != nil { iterator.Release() return 0, fmt.Errorf("PruneOrphaned: error getting instance account: %w", err) @@ -223,7 +223,7 @@ func (m *manager) orphaned(ctx context.Context, key string, instanceAccountID st // Look for keys in storage that we don't have an attachment for. switch Type(mediaType) { case TypeAttachment, TypeHeader, TypeAvatar: - if _, err := m.db.GetAttachmentByID(ctx, mediaID); err != nil { + if _, err := m.state.DB.GetAttachmentByID(ctx, mediaID); err != nil { if !errors.Is(err, db.ErrNoEntries) { return false, fmt.Errorf("error calling GetAttachmentByID: %w", err) } @@ -234,7 +234,7 @@ func (m *manager) orphaned(ctx context.Context, key string, instanceAccountID st // the MEDIA_ID part of the key for emojis will not necessarily correspond // to the file that's currently being used as the emoji image. staticURL := uris.GenerateURIForAttachment(instanceAccountID, string(TypeEmoji), string(SizeStatic), mediaID, mimePng) - if _, err := m.db.GetEmojiByStaticURL(ctx, staticURL); err != nil { + if _, err := m.state.DB.GetEmojiByStaticURL(ctx, staticURL); err != nil { if !errors.Is(err, db.ErrNoEntries) { return false, fmt.Errorf("error calling GetEmojiByStaticURL: %w", err) } @@ -254,7 +254,7 @@ func (m *manager) UncacheRemote(ctx context.Context, olderThanDays int, dry bool if dry { // Dry run, just count eligible entries without removing them. - return m.db.CountRemoteOlderThan(ctx, olderThan) + return m.state.DB.CountRemoteOlderThan(ctx, olderThan) } var ( @@ -263,7 +263,7 @@ func (m *manager) UncacheRemote(ctx context.Context, olderThanDays int, dry bool err error ) - for attachments, err = m.db.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.db.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit) { + for attachments, err = m.state.DB.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.state.DB.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit) { olderThan = attachments[len(attachments)-1].CreatedAt // use the created time of the last attachment in the slice as the next 'olderThan' value for _, attachment := range attachments { @@ -287,7 +287,7 @@ func (m *manager) PruneUnusedLocal(ctx context.Context, dry bool) (int, error) { if dry { // Dry run, just count eligible entries without removing them. - return m.db.CountLocalUnattachedOlderThan(ctx, olderThan) + return m.state.DB.CountLocalUnattachedOlderThan(ctx, olderThan) } var ( @@ -296,7 +296,7 @@ func (m *manager) PruneUnusedLocal(ctx context.Context, dry bool) (int, error) { err error ) - for attachments, err = m.db.GetLocalUnattachedOlderThan(ctx, olderThan, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.db.GetLocalUnattachedOlderThan(ctx, olderThan, selectPruneLimit) { + for attachments, err = m.state.DB.GetLocalUnattachedOlderThan(ctx, olderThan, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.state.DB.GetLocalUnattachedOlderThan(ctx, olderThan, selectPruneLimit) { olderThan = attachments[len(attachments)-1].CreatedAt // use the created time of the last attachment in the slice as the next 'olderThan' value for _, attachment := range attachments { @@ -325,7 +325,7 @@ func (m *manager) deleteAttachment(ctx context.Context, attachment *gtsmodel.Med } // Delete attachment completely. - return m.db.DeleteByID(ctx, attachment.ID, attachment) + return m.state.DB.DeleteByID(ctx, attachment.ID, attachment) } func (m *manager) uncacheAttachment(ctx context.Context, attachment *gtsmodel.MediaAttachment) error { @@ -337,14 +337,14 @@ func (m *manager) uncacheAttachment(ctx context.Context, attachment *gtsmodel.Me attachment.UpdatedAt = time.Now() cached := false attachment.Cached = &cached - return m.db.UpdateByID(ctx, attachment, attachment.ID, "updated_at", "cached") + return m.state.DB.UpdateByID(ctx, attachment, attachment.ID, "updated_at", "cached") } func (m *manager) removeFiles(ctx context.Context, keys ...string) error { errs := make(gtserror.MultiError, 0, len(keys)) for _, key := range keys { - if err := m.storage.Delete(ctx, key); err != nil && !errors.Is(err, storage.ErrNotFound) { + if err := m.state.Storage.Delete(ctx, key); err != nil && !errors.Is(err, storage.ErrNotFound) { errs = append(errs, "storage error removing "+key+": "+err.Error()) } } diff --git a/internal/media/prune_test.go b/internal/media/prune_test.go index ed040913f..d96c60ef5 100644 --- a/internal/media/prune_test.go +++ b/internal/media/prune_test.go @@ -313,7 +313,7 @@ func (suite *PruneTestSuite) TestUncacheAndRecache() { testStatusAttachment, testHeader, } { - processingRecache, err := suite.manager.RecacheMedia(ctx, data, nil, original.ID) + processingRecache, err := suite.manager.PreProcessMediaRecache(ctx, data, nil, original.ID) suite.NoError(err) // synchronously load the recached attachment diff --git a/internal/media/refetch.go b/internal/media/refetch.go index e29b4587f..3d572e4b9 100644 --- a/internal/media/refetch.go +++ b/internal/media/refetch.go @@ -47,7 +47,7 @@ func (m *manager) RefetchEmojis(ctx context.Context, domain string, dereferenceM // page through emojis 20 at a time, looking for those with missing images for { // Fetch next block of emojis from database - emojis, err := m.db.GetEmojis(ctx, domain, false, true, "", maxShortcodeDomain, "", 20) + emojis, err := m.state.DB.GetEmojis(ctx, domain, false, true, "", maxShortcodeDomain, "", 20) if err != nil { if !errors.Is(err, db.ErrNoEntries) { // an actual error has occurred @@ -86,7 +86,7 @@ func (m *manager) RefetchEmojis(ctx context.Context, domain string, dereferenceM var totalRefetched int for _, emojiID := range refetchIDs { - emoji, err := m.db.GetEmojiByID(ctx, emojiID) + emoji, err := m.state.DB.GetEmojiByID(ctx, emojiID) if err != nil { // this shouldn't happen--since we know we have the emoji--so return if it does return 0, fmt.Errorf("error getting emoji %s: %w", emojiID, err) @@ -108,7 +108,7 @@ func (m *manager) RefetchEmojis(ctx context.Context, domain string, dereferenceM return dereferenceMedia(ctx, emojiImageIRI) } - processingEmoji, err := m.ProcessEmoji(ctx, dataFunc, nil, emoji.Shortcode, emoji.ID, emoji.URI, &AdditionalEmojiInfo{ + processingEmoji, err := m.PreProcessEmoji(ctx, dataFunc, nil, emoji.Shortcode, emoji.ID, emoji.URI, &AdditionalEmojiInfo{ Domain: &emoji.Domain, ImageRemoteURL: &emoji.ImageRemoteURL, ImageStaticRemoteURL: &emoji.ImageStaticRemoteURL, @@ -133,13 +133,13 @@ func (m *manager) RefetchEmojis(ctx context.Context, domain string, dereferenceM } func (m *manager) emojiRequiresRefetch(ctx context.Context, emoji *gtsmodel.Emoji) (bool, error) { - if has, err := m.storage.Has(ctx, emoji.ImagePath); err != nil { + if has, err := m.state.Storage.Has(ctx, emoji.ImagePath); err != nil { return false, err } else if !has { return true, nil } - if has, err := m.storage.Has(ctx, emoji.ImageStaticPath); err != nil { + if has, err := m.state.Storage.Has(ctx, emoji.ImageStaticPath); err != nil { return false, err } else if !has { return true, nil diff --git a/internal/processing/account/update.go b/internal/processing/account/update.go index 370e661ed..dc4081915 100644 --- a/internal/processing/account/update.go +++ b/internal/processing/account/update.go @@ -194,7 +194,7 @@ func (p *processor) UpdateAvatar(ctx context.Context, avatar *multipart.FileHead Description: description, } - processingMedia, err := p.mediaManager.ProcessMedia(ctx, dataFunc, nil, accountID, ai) + processingMedia, err := p.mediaManager.PreProcessMedia(ctx, dataFunc, nil, accountID, ai) if err != nil { return nil, fmt.Errorf("UpdateAvatar: error processing avatar: %s", err) } @@ -221,10 +221,7 @@ func (p *processor) UpdateHeader(ctx context.Context, header *multipart.FileHead Header: &isHeader, } - processingMedia, err := p.mediaManager.ProcessMedia(ctx, dataFunc, nil, accountID, ai) - if err != nil { - return nil, fmt.Errorf("UpdateHeader: error processing header: %s", err) - } + processingMedia, err := p.mediaManager.PreProcessMedia(ctx, dataFunc, nil, accountID, ai) if err != nil { return nil, fmt.Errorf("UpdateHeader: error processing header: %s", err) } diff --git a/internal/processing/admin/createemoji.go b/internal/processing/admin/createemoji.go index 601ced336..b2a7bfc86 100644 --- a/internal/processing/admin/createemoji.go +++ b/internal/processing/admin/createemoji.go @@ -70,7 +70,7 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, } } - processingEmoji, err := p.mediaManager.ProcessEmoji(ctx, data, nil, form.Shortcode, emojiID, emojiURI, ai, false) + processingEmoji, err := p.mediaManager.PreProcessEmoji(ctx, data, nil, form.Shortcode, emojiID, emojiURI, ai, false) if err != nil { return nil, gtserror.NewErrorInternalError(fmt.Errorf("error processing emoji: %s", err), "error processing emoji") } diff --git a/internal/processing/admin/updateemoji.go b/internal/processing/admin/updateemoji.go index 370e6e27f..41ccd609c 100644 --- a/internal/processing/admin/updateemoji.go +++ b/internal/processing/admin/updateemoji.go @@ -107,7 +107,7 @@ func (p *processor) emojiUpdateCopy(ctx context.Context, emoji *gtsmodel.Emoji, } } - processingEmoji, err := p.mediaManager.ProcessEmoji(ctx, data, nil, *shortcode, newEmojiID, newEmojiURI, ai, false) + processingEmoji, err := p.mediaManager.PreProcessEmoji(ctx, data, nil, *shortcode, newEmojiID, newEmojiURI, ai, false) if err != nil { err = fmt.Errorf("emojiUpdateCopy: error processing emoji %s: %s", emoji.ID, err) return nil, gtserror.NewErrorInternalError(err) @@ -213,7 +213,7 @@ func (p *processor) emojiUpdateModify(ctx context.Context, emoji *gtsmodel.Emoji } } - processingEmoji, err := p.mediaManager.ProcessEmoji(ctx, data, nil, emoji.Shortcode, emoji.ID, emoji.URI, ai, true) + processingEmoji, err := p.mediaManager.PreProcessEmoji(ctx, data, nil, emoji.Shortcode, emoji.ID, emoji.URI, ai, true) if err != nil { err = fmt.Errorf("emojiUpdateModify: error processing emoji %s: %s", emoji.ID, err) return nil, gtserror.NewErrorInternalError(err) diff --git a/internal/processing/media/create.go b/internal/processing/media/create.go index dbe22fe33..494434acb 100644 --- a/internal/processing/media/create.go +++ b/internal/processing/media/create.go @@ -42,7 +42,7 @@ func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form } // process the media attachment and load it immediately - media, err := p.mediaManager.ProcessMedia(ctx, data, nil, account.ID, &media.AdditionalMediaInfo{ + media, err := p.mediaManager.PreProcessMedia(ctx, data, nil, account.ID, &media.AdditionalMediaInfo{ Description: &form.Description, FocusX: &focusX, FocusY: &focusY, diff --git a/internal/processing/media/getfile.go b/internal/processing/media/getfile.go index 9843d6714..41250b3f5 100644 --- a/internal/processing/media/getfile.go +++ b/internal/processing/media/getfile.go @@ -159,7 +159,7 @@ func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount } // Start recaching this media with the prepared data function. - processingMedia, err := p.mediaManager.RecacheMedia(ctx, dataFn, nil, wantedMediaID) + processingMedia, err := p.mediaManager.PreProcessMediaRecache(ctx, dataFn, nil, wantedMediaID) if err != nil { return nil, gtserror.NewErrorNotFound(fmt.Errorf("error recaching media: %s", err)) } diff --git a/internal/router/router.go b/internal/router/router.go index 898f145fe..5225214ab 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -117,7 +117,7 @@ func (r *router) Start() { // "debug" or "debugenv" build-tag is set pprof stats will be served // at the standard "/debug/pprof" URL. r.srv.Handler = debug.WithPprof(r.srv.Handler) - if debug.DEBUG() { + if debug.DEBUG { // Profiling requires timeouts longer than 30s, so reset these. log.Warn("resetting http.Server{} timeout to support profiling") r.srv.ReadTimeout = 0 diff --git a/internal/state/state.go b/internal/state/state.go index 479e9f56f..f3a907378 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -21,6 +21,8 @@ package state import ( "github.com/superseriousbusiness/gotosocial/internal/cache" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/storage" + "github.com/superseriousbusiness/gotosocial/internal/workers" ) // State provides a means of dependency injection and sharing of resources @@ -36,6 +38,12 @@ type State struct { // DB provides access to the database. DB db.DB + // Storage provides access to the storage driver. + Storage *storage.Driver + + // Workers provides access to this state's collection of worker pools. + Workers workers.Workers + // prevent pass-by-value. _ nocopy } diff --git a/internal/workers/workers.go b/internal/workers/workers.go new file mode 100644 index 000000000..77b3065ce --- /dev/null +++ b/internal/workers/workers.go @@ -0,0 +1,90 @@ +/* +GoToSocial +Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package workers + +import ( + "log" + "runtime" + + "codeberg.org/gruf/go-runners" + "codeberg.org/gruf/go-sched" +) + +type Workers struct { + // Main task scheduler instance. + Scheduler sched.Scheduler + + // Processor / federator worker pools. + // ClientAPI runners.WorkerPool + // Federator runners.WorkerPool + + // Media manager worker pools. + Media runners.WorkerPool + + // prevent pass-by-value. + _ nocopy +} + +// Start will start all of the contained worker pools (and global scheduler). +func (w *Workers) Start() { + // Get currently set GOMAXPROCS. + maxprocs := runtime.GOMAXPROCS(0) + + tryUntil("starting scheduler", 5, func() bool { + return w.Scheduler.Start(nil) + }) + + // tryUntil("starting client API workerpool", 5, func() bool { + // return w.ClientAPI.Start(4*maxprocs, 400*maxprocs) + // }) + + // tryUntil("starting federator workerpool", 5, func() bool { + // return w.Federator.Start(4*maxprocs, 400*maxprocs) + // }) + + tryUntil("starting media workerpool", 5, func() bool { + return w.Media.Start(8*maxprocs, 80*maxprocs) + }) +} + +// Stop will stop all of the contained worker pools (and global scheduler). +func (w *Workers) Stop() { + tryUntil("stopping scheduler", 5, w.Scheduler.Stop) + // tryUntil("stopping client API workerpool", 5, w.ClientAPI.Stop) + // tryUntil("stopping federator workerpool", 5, w.Federator.Stop) + tryUntil("stopping media workerpool", 5, w.Media.Stop) +} + +// nocopy when embedded will signal linter to +// error on pass-by-value of parent struct. +type nocopy struct{} + +func (*nocopy) Lock() {} + +func (*nocopy) Unlock() {} + +// tryUntil will attempt to call 'do' for 'count' attempts, before panicking with 'msg'. +func tryUntil(msg string, count int, do func() bool) { + for i := 0; i < count; i++ { + if do() { + return + } + } + log.Panicf("failed %s after %d tries", msg, count) +} diff --git a/testrig/federatingdb.go b/testrig/federatingdb.go index f9d47979a..9b1f1961e 100644 --- a/testrig/federatingdb.go +++ b/testrig/federatingdb.go @@ -27,5 +27,5 @@ import ( // NewTestFederatingDB returns a federating DB with the underlying db func NewTestFederatingDB(db db.DB, fedWorker *concurrency.WorkerPool[messages.FromFederator]) federatingdb.DB { - return federatingdb.New(db, fedWorker) + return federatingdb.New(db, fedWorker, NewTestTypeConverter(db)) } diff --git a/testrig/mediahandler.go b/testrig/mediahandler.go index cb87b598e..a1863218c 100644 --- a/testrig/mediahandler.go +++ b/testrig/mediahandler.go @@ -21,14 +21,15 @@ package testrig import ( "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/state" "github.com/superseriousbusiness/gotosocial/internal/storage" ) // NewTestMediaManager returns a media handler with the default test config, and the given db and storage. func NewTestMediaManager(db db.DB, storage *storage.Driver) media.Manager { - m, err := media.NewManager(db, storage) - if err != nil { - panic(err) - } - return m + var state state.State + state.DB = db + state.Storage = storage + state.Workers.Start() + return media.NewManager(&state) } diff --git a/vendor/codeberg.org/gruf/go-bitutil/abs.go b/vendor/codeberg.org/gruf/go-bitutil/abs.go new file mode 100644 index 000000000..f4ce8ad75 --- /dev/null +++ b/vendor/codeberg.org/gruf/go-bitutil/abs.go @@ -0,0 +1,29 @@ +package bitutil + +// Abs8 returns the absolute value of i (calculated without branching). +func Abs8(i int8) int8 { + const bits = 8 + u := uint64(i >> (bits - 1)) + return (i ^ int8(u)) + int8(u&1) +} + +// Abs16 returns the absolute value of i (calculated without branching). +func Abs16(i int16) int16 { + const bits = 16 + u := uint64(i >> (bits - 1)) + return (i ^ int16(u)) + int16(u&1) +} + +// Abs32 returns the absolute value of i (calculated without branching). +func Abs32(i int32) int32 { + const bits = 32 + u := uint64(i >> (bits - 1)) + return (i ^ int32(u)) + int32(u&1) +} + +// Abs64 returns the absolute value of i (calculated without branching). +func Abs64(i int64) int64 { + const bits = 64 + u := uint64(i >> (bits - 1)) + return (i ^ int64(u)) + int64(u&1) +} diff --git a/vendor/codeberg.org/gruf/go-bitutil/flag.go b/vendor/codeberg.org/gruf/go-bitutil/flag.go index d8b0f8b66..6a5b20d11 100644 --- a/vendor/codeberg.org/gruf/go-bitutil/flag.go +++ b/vendor/codeberg.org/gruf/go-bitutil/flag.go @@ -1,11 +1,11 @@ package bitutil import ( - "codeberg.org/gruf/go-byteutil" + "unsafe" ) // Flags8 is a type-casted unsigned integer with helper -// methods for easily managing up to 8 bit flags. +// methods for easily managing up to 8 bit-flags. type Flags8 uint8 // Get will fetch the flag bit value at index 'bit'. @@ -172,80 +172,135 @@ func (f Flags8) Unset7() Flags8 { // String returns a human readable representation of Flags8. func (f Flags8) String() string { - var val bool - var buf byteutil.Buffer + var ( + i int + val bool + buf []byte + ) - buf.WriteByte('{') + // Make a prealloc est. based on longest-possible value + const prealloc = 1 + (len("false ") * 8) - 1 + 1 + buf = make([]byte, prealloc) + + buf[i] = '{' + i++ val = f.Get0() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get1() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get2() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get3() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get4() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get5() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get6() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get7() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ - buf.Truncate(1) - buf.WriteByte('}') + buf[i-1] = '}' + buf = buf[:i] - return buf.String() + return *(*string)(unsafe.Pointer(&buf)) } // GoString returns a more verbose human readable representation of Flags8. func (f Flags8) GoString() string { - var val bool - var buf byteutil.Buffer + var ( + i int + val bool + buf []byte + ) + + // Make a prealloc est. based on longest-possible value + const prealloc = len("bitutil.Flags8{") + (len("7=false ") * 8) - 1 + 1 + buf = make([]byte, prealloc) - buf.WriteString("bitutil.Flags8{") + i += copy(buf[i:], "bitutil.Flags8{") val = f.Get0() - buf.WriteString("0=" + bool2str(val) + " ") + i += copy(buf[i:], "0=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get1() - buf.WriteString("1=" + bool2str(val) + " ") + i += copy(buf[i:], "1=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get2() - buf.WriteString("2=" + bool2str(val) + " ") + i += copy(buf[i:], "2=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get3() - buf.WriteString("3=" + bool2str(val) + " ") + i += copy(buf[i:], "3=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get4() - buf.WriteString("4=" + bool2str(val) + " ") + i += copy(buf[i:], "4=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get5() - buf.WriteString("5=" + bool2str(val) + " ") + i += copy(buf[i:], "5=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get6() - buf.WriteString("6=" + bool2str(val) + " ") + i += copy(buf[i:], "6=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get7() - buf.WriteString("7=" + bool2str(val) + " ") + i += copy(buf[i:], "7=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ - buf.Truncate(1) - buf.WriteByte('}') + buf[i-1] = '}' + buf = buf[:i] - return buf.String() + return *(*string)(unsafe.Pointer(&buf)) } // Flags16 is a type-casted unsigned integer with helper -// methods for easily managing up to 16 bit flags. +// methods for easily managing up to 16 bit-flags. type Flags16 uint16 // Get will fetch the flag bit value at index 'bit'. @@ -556,128 +611,223 @@ func (f Flags16) Unset15() Flags16 { // String returns a human readable representation of Flags16. func (f Flags16) String() string { - var val bool - var buf byteutil.Buffer + var ( + i int + val bool + buf []byte + ) - buf.WriteByte('{') + // Make a prealloc est. based on longest-possible value + const prealloc = 1 + (len("false ") * 16) - 1 + 1 + buf = make([]byte, prealloc) + + buf[i] = '{' + i++ val = f.Get0() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get1() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get2() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get3() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get4() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get5() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get6() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get7() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get8() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get9() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get10() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get11() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get12() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get13() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get14() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get15() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ - buf.Truncate(1) - buf.WriteByte('}') + buf[i-1] = '}' + buf = buf[:i] - return buf.String() + return *(*string)(unsafe.Pointer(&buf)) } // GoString returns a more verbose human readable representation of Flags16. func (f Flags16) GoString() string { - var val bool - var buf byteutil.Buffer + var ( + i int + val bool + buf []byte + ) + + // Make a prealloc est. based on longest-possible value + const prealloc = len("bitutil.Flags16{") + (len("15=false ") * 16) - 1 + 1 + buf = make([]byte, prealloc) - buf.WriteString("bitutil.Flags16{") + i += copy(buf[i:], "bitutil.Flags16{") val = f.Get0() - buf.WriteString("0=" + bool2str(val) + " ") + i += copy(buf[i:], "0=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get1() - buf.WriteString("1=" + bool2str(val) + " ") + i += copy(buf[i:], "1=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get2() - buf.WriteString("2=" + bool2str(val) + " ") + i += copy(buf[i:], "2=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get3() - buf.WriteString("3=" + bool2str(val) + " ") + i += copy(buf[i:], "3=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get4() - buf.WriteString("4=" + bool2str(val) + " ") + i += copy(buf[i:], "4=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get5() - buf.WriteString("5=" + bool2str(val) + " ") + i += copy(buf[i:], "5=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get6() - buf.WriteString("6=" + bool2str(val) + " ") + i += copy(buf[i:], "6=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get7() - buf.WriteString("7=" + bool2str(val) + " ") + i += copy(buf[i:], "7=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get8() - buf.WriteString("8=" + bool2str(val) + " ") + i += copy(buf[i:], "8=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get9() - buf.WriteString("9=" + bool2str(val) + " ") + i += copy(buf[i:], "9=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get10() - buf.WriteString("10=" + bool2str(val) + " ") + i += copy(buf[i:], "10=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get11() - buf.WriteString("11=" + bool2str(val) + " ") + i += copy(buf[i:], "11=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get12() - buf.WriteString("12=" + bool2str(val) + " ") + i += copy(buf[i:], "12=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get13() - buf.WriteString("13=" + bool2str(val) + " ") + i += copy(buf[i:], "13=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get14() - buf.WriteString("14=" + bool2str(val) + " ") + i += copy(buf[i:], "14=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get15() - buf.WriteString("15=" + bool2str(val) + " ") + i += copy(buf[i:], "15=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ - buf.Truncate(1) - buf.WriteByte('}') + buf[i-1] = '}' + buf = buf[:i] - return buf.String() + return *(*string)(unsafe.Pointer(&buf)) } // Flags32 is a type-casted unsigned integer with helper -// methods for easily managing up to 32 bit flags. +// methods for easily managing up to 32 bit-flags. type Flags32 uint32 // Get will fetch the flag bit value at index 'bit'. @@ -1276,224 +1426,399 @@ func (f Flags32) Unset31() Flags32 { // String returns a human readable representation of Flags32. func (f Flags32) String() string { - var val bool - var buf byteutil.Buffer + var ( + i int + val bool + buf []byte + ) - buf.WriteByte('{') + // Make a prealloc est. based on longest-possible value + const prealloc = 1 + (len("false ") * 32) - 1 + 1 + buf = make([]byte, prealloc) + + buf[i] = '{' + i++ val = f.Get0() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get1() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get2() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get3() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get4() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get5() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get6() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get7() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get8() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get9() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get10() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get11() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get12() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get13() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get14() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get15() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get16() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get17() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get18() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get19() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get20() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get21() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get22() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get23() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get24() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get25() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get26() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get27() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get28() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get29() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get30() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get31() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ - buf.Truncate(1) - buf.WriteByte('}') + buf[i-1] = '}' + buf = buf[:i] - return buf.String() + return *(*string)(unsafe.Pointer(&buf)) } // GoString returns a more verbose human readable representation of Flags32. func (f Flags32) GoString() string { - var val bool - var buf byteutil.Buffer + var ( + i int + val bool + buf []byte + ) + + // Make a prealloc est. based on longest-possible value + const prealloc = len("bitutil.Flags32{") + (len("31=false ") * 32) - 1 + 1 + buf = make([]byte, prealloc) - buf.WriteString("bitutil.Flags32{") + i += copy(buf[i:], "bitutil.Flags32{") val = f.Get0() - buf.WriteString("0=" + bool2str(val) + " ") + i += copy(buf[i:], "0=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get1() - buf.WriteString("1=" + bool2str(val) + " ") + i += copy(buf[i:], "1=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get2() - buf.WriteString("2=" + bool2str(val) + " ") + i += copy(buf[i:], "2=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get3() - buf.WriteString("3=" + bool2str(val) + " ") + i += copy(buf[i:], "3=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get4() - buf.WriteString("4=" + bool2str(val) + " ") + i += copy(buf[i:], "4=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get5() - buf.WriteString("5=" + bool2str(val) + " ") + i += copy(buf[i:], "5=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get6() - buf.WriteString("6=" + bool2str(val) + " ") + i += copy(buf[i:], "6=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get7() - buf.WriteString("7=" + bool2str(val) + " ") + i += copy(buf[i:], "7=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get8() - buf.WriteString("8=" + bool2str(val) + " ") + i += copy(buf[i:], "8=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get9() - buf.WriteString("9=" + bool2str(val) + " ") + i += copy(buf[i:], "9=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get10() - buf.WriteString("10=" + bool2str(val) + " ") + i += copy(buf[i:], "10=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get11() - buf.WriteString("11=" + bool2str(val) + " ") + i += copy(buf[i:], "11=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get12() - buf.WriteString("12=" + bool2str(val) + " ") + i += copy(buf[i:], "12=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get13() - buf.WriteString("13=" + bool2str(val) + " ") + i += copy(buf[i:], "13=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get14() - buf.WriteString("14=" + bool2str(val) + " ") + i += copy(buf[i:], "14=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get15() - buf.WriteString("15=" + bool2str(val) + " ") + i += copy(buf[i:], "15=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get16() - buf.WriteString("16=" + bool2str(val) + " ") + i += copy(buf[i:], "16=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get17() - buf.WriteString("17=" + bool2str(val) + " ") + i += copy(buf[i:], "17=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get18() - buf.WriteString("18=" + bool2str(val) + " ") + i += copy(buf[i:], "18=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get19() - buf.WriteString("19=" + bool2str(val) + " ") + i += copy(buf[i:], "19=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get20() - buf.WriteString("20=" + bool2str(val) + " ") + i += copy(buf[i:], "20=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get21() - buf.WriteString("21=" + bool2str(val) + " ") + i += copy(buf[i:], "21=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get22() - buf.WriteString("22=" + bool2str(val) + " ") + i += copy(buf[i:], "22=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get23() - buf.WriteString("23=" + bool2str(val) + " ") + i += copy(buf[i:], "23=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get24() - buf.WriteString("24=" + bool2str(val) + " ") + i += copy(buf[i:], "24=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get25() - buf.WriteString("25=" + bool2str(val) + " ") + i += copy(buf[i:], "25=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get26() - buf.WriteString("26=" + bool2str(val) + " ") + i += copy(buf[i:], "26=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get27() - buf.WriteString("27=" + bool2str(val) + " ") + i += copy(buf[i:], "27=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get28() - buf.WriteString("28=" + bool2str(val) + " ") + i += copy(buf[i:], "28=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get29() - buf.WriteString("29=" + bool2str(val) + " ") + i += copy(buf[i:], "29=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get30() - buf.WriteString("30=" + bool2str(val) + " ") + i += copy(buf[i:], "30=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get31() - buf.WriteString("31=" + bool2str(val) + " ") + i += copy(buf[i:], "31=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ - buf.Truncate(1) - buf.WriteByte('}') + buf[i-1] = '}' + buf = buf[:i] - return buf.String() + return *(*string)(unsafe.Pointer(&buf)) } // Flags64 is a type-casted unsigned integer with helper -// methods for easily managing up to 64 bit flags. +// methods for easily managing up to 64 bit-flags. type Flags64 uint64 // Get will fetch the flag bit value at index 'bit'. @@ -2668,412 +2993,747 @@ func (f Flags64) Unset63() Flags64 { // String returns a human readable representation of Flags64. func (f Flags64) String() string { - var val bool - var buf byteutil.Buffer + var ( + i int + val bool + buf []byte + ) - buf.WriteByte('{') + // Make a prealloc est. based on longest-possible value + const prealloc = 1 + (len("false ") * 64) - 1 + 1 + buf = make([]byte, prealloc) + + buf[i] = '{' + i++ val = f.Get0() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get1() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get2() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get3() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get4() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get5() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get6() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get7() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get8() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get9() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get10() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get11() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get12() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get13() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get14() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get15() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get16() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get17() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get18() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get19() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get20() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get21() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get22() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get23() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get24() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get25() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get26() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get27() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get28() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get29() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get30() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get31() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get32() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get33() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get34() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get35() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get36() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get37() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get38() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get39() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get40() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get41() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get42() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get43() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get44() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get45() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get46() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get47() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get48() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get49() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get50() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get51() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get52() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get53() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get54() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get55() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get56() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get57() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get58() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get59() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get60() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get61() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get62() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get63() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ - buf.Truncate(1) - buf.WriteByte('}') + buf[i-1] = '}' + buf = buf[:i] - return buf.String() + return *(*string)(unsafe.Pointer(&buf)) } // GoString returns a more verbose human readable representation of Flags64. func (f Flags64) GoString() string { - var val bool - var buf byteutil.Buffer + var ( + i int + val bool + buf []byte + ) + + // Make a prealloc est. based on longest-possible value + const prealloc = len("bitutil.Flags64{") + (len("63=false ") * 64) - 1 + 1 + buf = make([]byte, prealloc) - buf.WriteString("bitutil.Flags64{") + i += copy(buf[i:], "bitutil.Flags64{") val = f.Get0() - buf.WriteString("0=" + bool2str(val) + " ") + i += copy(buf[i:], "0=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get1() - buf.WriteString("1=" + bool2str(val) + " ") + i += copy(buf[i:], "1=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get2() - buf.WriteString("2=" + bool2str(val) + " ") + i += copy(buf[i:], "2=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get3() - buf.WriteString("3=" + bool2str(val) + " ") + i += copy(buf[i:], "3=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get4() - buf.WriteString("4=" + bool2str(val) + " ") + i += copy(buf[i:], "4=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get5() - buf.WriteString("5=" + bool2str(val) + " ") + i += copy(buf[i:], "5=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get6() - buf.WriteString("6=" + bool2str(val) + " ") + i += copy(buf[i:], "6=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get7() - buf.WriteString("7=" + bool2str(val) + " ") + i += copy(buf[i:], "7=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get8() - buf.WriteString("8=" + bool2str(val) + " ") + i += copy(buf[i:], "8=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get9() - buf.WriteString("9=" + bool2str(val) + " ") + i += copy(buf[i:], "9=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get10() - buf.WriteString("10=" + bool2str(val) + " ") + i += copy(buf[i:], "10=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get11() - buf.WriteString("11=" + bool2str(val) + " ") + i += copy(buf[i:], "11=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get12() - buf.WriteString("12=" + bool2str(val) + " ") + i += copy(buf[i:], "12=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get13() - buf.WriteString("13=" + bool2str(val) + " ") + i += copy(buf[i:], "13=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get14() - buf.WriteString("14=" + bool2str(val) + " ") + i += copy(buf[i:], "14=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get15() - buf.WriteString("15=" + bool2str(val) + " ") + i += copy(buf[i:], "15=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get16() - buf.WriteString("16=" + bool2str(val) + " ") + i += copy(buf[i:], "16=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get17() - buf.WriteString("17=" + bool2str(val) + " ") + i += copy(buf[i:], "17=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get18() - buf.WriteString("18=" + bool2str(val) + " ") + i += copy(buf[i:], "18=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get19() - buf.WriteString("19=" + bool2str(val) + " ") + i += copy(buf[i:], "19=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get20() - buf.WriteString("20=" + bool2str(val) + " ") + i += copy(buf[i:], "20=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get21() - buf.WriteString("21=" + bool2str(val) + " ") + i += copy(buf[i:], "21=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get22() - buf.WriteString("22=" + bool2str(val) + " ") + i += copy(buf[i:], "22=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get23() - buf.WriteString("23=" + bool2str(val) + " ") + i += copy(buf[i:], "23=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get24() - buf.WriteString("24=" + bool2str(val) + " ") + i += copy(buf[i:], "24=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get25() - buf.WriteString("25=" + bool2str(val) + " ") + i += copy(buf[i:], "25=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get26() - buf.WriteString("26=" + bool2str(val) + " ") + i += copy(buf[i:], "26=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get27() - buf.WriteString("27=" + bool2str(val) + " ") + i += copy(buf[i:], "27=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get28() - buf.WriteString("28=" + bool2str(val) + " ") + i += copy(buf[i:], "28=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get29() - buf.WriteString("29=" + bool2str(val) + " ") + i += copy(buf[i:], "29=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get30() - buf.WriteString("30=" + bool2str(val) + " ") + i += copy(buf[i:], "30=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get31() - buf.WriteString("31=" + bool2str(val) + " ") + i += copy(buf[i:], "31=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get32() - buf.WriteString("32=" + bool2str(val) + " ") + i += copy(buf[i:], "32=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get33() - buf.WriteString("33=" + bool2str(val) + " ") + i += copy(buf[i:], "33=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get34() - buf.WriteString("34=" + bool2str(val) + " ") + i += copy(buf[i:], "34=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get35() - buf.WriteString("35=" + bool2str(val) + " ") + i += copy(buf[i:], "35=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get36() - buf.WriteString("36=" + bool2str(val) + " ") + i += copy(buf[i:], "36=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get37() - buf.WriteString("37=" + bool2str(val) + " ") + i += copy(buf[i:], "37=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get38() - buf.WriteString("38=" + bool2str(val) + " ") + i += copy(buf[i:], "38=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get39() - buf.WriteString("39=" + bool2str(val) + " ") + i += copy(buf[i:], "39=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get40() - buf.WriteString("40=" + bool2str(val) + " ") + i += copy(buf[i:], "40=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get41() - buf.WriteString("41=" + bool2str(val) + " ") + i += copy(buf[i:], "41=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get42() - buf.WriteString("42=" + bool2str(val) + " ") + i += copy(buf[i:], "42=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get43() - buf.WriteString("43=" + bool2str(val) + " ") + i += copy(buf[i:], "43=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get44() - buf.WriteString("44=" + bool2str(val) + " ") + i += copy(buf[i:], "44=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get45() - buf.WriteString("45=" + bool2str(val) + " ") + i += copy(buf[i:], "45=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get46() - buf.WriteString("46=" + bool2str(val) + " ") + i += copy(buf[i:], "46=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get47() - buf.WriteString("47=" + bool2str(val) + " ") + i += copy(buf[i:], "47=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get48() - buf.WriteString("48=" + bool2str(val) + " ") + i += copy(buf[i:], "48=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get49() - buf.WriteString("49=" + bool2str(val) + " ") + i += copy(buf[i:], "49=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get50() - buf.WriteString("50=" + bool2str(val) + " ") + i += copy(buf[i:], "50=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get51() - buf.WriteString("51=" + bool2str(val) + " ") + i += copy(buf[i:], "51=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get52() - buf.WriteString("52=" + bool2str(val) + " ") + i += copy(buf[i:], "52=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get53() - buf.WriteString("53=" + bool2str(val) + " ") + i += copy(buf[i:], "53=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get54() - buf.WriteString("54=" + bool2str(val) + " ") + i += copy(buf[i:], "54=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get55() - buf.WriteString("55=" + bool2str(val) + " ") + i += copy(buf[i:], "55=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get56() - buf.WriteString("56=" + bool2str(val) + " ") + i += copy(buf[i:], "56=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get57() - buf.WriteString("57=" + bool2str(val) + " ") + i += copy(buf[i:], "57=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get58() - buf.WriteString("58=" + bool2str(val) + " ") + i += copy(buf[i:], "58=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get59() - buf.WriteString("59=" + bool2str(val) + " ") + i += copy(buf[i:], "59=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get60() - buf.WriteString("60=" + bool2str(val) + " ") + i += copy(buf[i:], "60=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get61() - buf.WriteString("61=" + bool2str(val) + " ") + i += copy(buf[i:], "61=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get62() - buf.WriteString("62=" + bool2str(val) + " ") + i += copy(buf[i:], "62=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ val = f.Get63() - buf.WriteString("63=" + bool2str(val) + " ") + i += copy(buf[i:], "63=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ - buf.Truncate(1) - buf.WriteByte('}') + buf[i-1] = '}' + buf = buf[:i] - return buf.String() + return *(*string)(unsafe.Pointer(&buf)) } func bool2str(b bool) string { diff --git a/vendor/codeberg.org/gruf/go-bitutil/flag.tpl b/vendor/codeberg.org/gruf/go-bitutil/flag.tpl index 89f881930..ac00bfa97 100644 --- a/vendor/codeberg.org/gruf/go-bitutil/flag.tpl +++ b/vendor/codeberg.org/gruf/go-bitutil/flag.tpl @@ -2,14 +2,13 @@ package bitutil import ( "strings" - - "codeberg.org/gruf/go-byteutil" + "unsafe" ) {{ range $idx, $size := . }} // Flags{{ $size.Size }} is a type-casted unsigned integer with helper -// methods for easily managing up to {{ $size.Size }} bit flags. +// methods for easily managing up to {{ $size.Size }} bit-flags. type Flags{{ $size.Size }} uint{{ $size.Size }} // Get will fetch the flag bit value at index 'bit'. @@ -54,34 +53,58 @@ func (f Flags{{ $size.Size }}) Unset{{ $idx }}() Flags{{ $size.Size }} { // String returns a human readable representation of Flags{{ $size.Size }}. func (f Flags{{ $size.Size }}) String() string { - var val bool - var buf byteutil.Buffer + var ( + i int + val bool + buf []byte + ) + + // Make a prealloc est. based on longest-possible value + const prealloc = 1+(len("false ")*{{ $size.Size }})-1+1 + buf = make([]byte, prealloc) + + buf[i] = '{' + i++ - buf.WriteByte('{') {{ range $idx := .Bits }} val = f.Get{{ $idx }}() - buf.WriteString(bool2str(val) + " ") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ {{ end }} - buf.Truncate(1) - buf.WriteByte('}') - return buf.String() + buf[i-1] = '}' + buf = buf[:i] + + return *(*string)(unsafe.Pointer(&buf)) } // GoString returns a more verbose human readable representation of Flags{{ $size.Size }}. func (f Flags{{ $size.Size }})GoString() string { - var val bool - var buf byteutil.Buffer + var ( + i int + val bool + buf []byte + ) + + // Make a prealloc est. based on longest-possible value + const prealloc = len("bitutil.Flags{{ $size.Size }}{")+(len("{{ sub $size.Size 1 }}=false ")*{{ $size.Size }})-1+1 + buf = make([]byte, prealloc) + + i += copy(buf[i:], "bitutil.Flags{{ $size.Size }}{") - buf.WriteString("bitutil.Flags{{ $size.Size }}{") {{ range $idx := .Bits }} val = f.Get{{ $idx }}() - buf.WriteString("{{ $idx }}="+bool2str(val)+" ") + i += copy(buf[i:], "{{ $idx }}=") + i += copy(buf[i:], bool2str(val)) + buf[i] = ' ' + i++ {{ end }} - buf.Truncate(1) - buf.WriteByte('}') + + buf[i-1] = '}' + buf = buf[:i] - return buf.String() + return *(*string)(unsafe.Pointer(&buf)) } {{ end }} diff --git a/vendor/codeberg.org/gruf/go-debug/debug.go b/vendor/codeberg.org/gruf/go-debug/debug.go index 6b5e56548..e7121390d 100644 --- a/vendor/codeberg.org/gruf/go-debug/debug.go +++ b/vendor/codeberg.org/gruf/go-debug/debug.go @@ -4,14 +4,9 @@ import ( _debug "runtime/debug" ) -// DEBUG returns whether debugging is enabled. -func DEBUG() bool { - return debug -} - // Run will only call fn if DEBUG is enabled. func Run(fn func()) { - if debug { + if DEBUG { fn() } } diff --git a/vendor/codeberg.org/gruf/go-debug/debug_env.go b/vendor/codeberg.org/gruf/go-debug/debug_env.go index 7ab231b08..4401b9725 100644 --- a/vendor/codeberg.org/gruf/go-debug/debug_env.go +++ b/vendor/codeberg.org/gruf/go-debug/debug_env.go @@ -5,5 +5,5 @@ package debug import "os" -// check if debug env variable is set -var debug = (os.Getenv("DEBUG") != "") +// DEBUG returns whether debugging is enabled. +var DEBUG = (os.Getenv("DEBUG") != "") diff --git a/vendor/codeberg.org/gruf/go-debug/debug_off.go b/vendor/codeberg.org/gruf/go-debug/debug_off.go index a7eb9daac..82ef7263e 100644 --- a/vendor/codeberg.org/gruf/go-debug/debug_off.go +++ b/vendor/codeberg.org/gruf/go-debug/debug_off.go @@ -3,5 +3,5 @@ package debug -// debug always off. -const debug = false +// DEBUG returns whether debugging is enabled. +const DEBUG = false diff --git a/vendor/codeberg.org/gruf/go-debug/debug_on.go b/vendor/codeberg.org/gruf/go-debug/debug_on.go index 744d70178..91edd5fe5 100644 --- a/vendor/codeberg.org/gruf/go-debug/debug_on.go +++ b/vendor/codeberg.org/gruf/go-debug/debug_on.go @@ -3,5 +3,5 @@ package debug -// debug always on. -const debug = true +// DEBUG returns whether debugging is enabled. +const DEBUG = true diff --git a/vendor/codeberg.org/gruf/go-debug/pprof_on.go b/vendor/codeberg.org/gruf/go-debug/pprof_on.go index a569ab823..4f91aa092 100644 --- a/vendor/codeberg.org/gruf/go-debug/pprof_on.go +++ b/vendor/codeberg.org/gruf/go-debug/pprof_on.go @@ -11,7 +11,7 @@ import ( // ServePprof will start an HTTP server serving /debug/pprof only if debug enabled. func ServePprof(addr string) error { - if !debug { + if !DEBUG { // debug disabled in env return nil } @@ -21,7 +21,7 @@ func ServePprof(addr string) error { // WithPprof will add /debug/pprof handling (provided by "net/http/pprof") only if debug enabled. func WithPprof(handler http.Handler) http.Handler { - if !debug { + if !DEBUG { // debug disabled in env return handler } diff --git a/vendor/codeberg.org/gruf/go-runners/pool.go b/vendor/codeberg.org/gruf/go-runners/pool.go index 16222b2e1..3d9105986 100644 --- a/vendor/codeberg.org/gruf/go-runners/pool.go +++ b/vendor/codeberg.org/gruf/go-runners/pool.go @@ -157,6 +157,34 @@ func (pool *WorkerPool) EnqueueCtx(ctx context.Context, fn WorkerFunc) bool { } } +// MustEnqueueCtx functionally performs similarly to WorkerPool.EnqueueCtx(), but in the case +// that the provided <-ctx.Done() is closed, it is passed asynchronously to WorkerPool.Enqueue(). +// Return boolean indicates whether function was executed in time before <-ctx.Done() is closed. +func (pool *WorkerPool) MustEnqueueCtx(ctx context.Context, fn WorkerFunc) (ok bool) { + // Check valid fn + if fn == nil { + return false + } + + select { + case <-ctx.Done(): + // We failed to add this entry to the worker queue before the + // incoming context was cancelled. So to ensure processing + // we simply queue it asynchronously and return early to caller. + go pool.Enqueue(fn) + return false + + case <-pool.svc.Done(): + // Pool ctx cancelled + fn(closedctx) + return false + + case pool.fns <- fn: + // Placed fn in queue + return true + } +} + // EnqueueNow attempts Enqueue but returns false if not executed. func (pool *WorkerPool) EnqueueNow(fn WorkerFunc) bool { // Check valid fn diff --git a/vendor/codeberg.org/gruf/go-runners/process.go b/vendor/codeberg.org/gruf/go-runners/process.go new file mode 100644 index 000000000..908e6edca --- /dev/null +++ b/vendor/codeberg.org/gruf/go-runners/process.go @@ -0,0 +1,75 @@ +package runners + +import ( + "fmt" + "sync" +) + +// Processable defines a runnable process with error return +// that can be passed to a Processor instance for managed running. +type Processable func() error + +// Processor acts similarly to a sync.Once object, except that it is reusable. After +// the first call to Process(), any further calls before this first has returned will +// block until the first call has returned, and return the same error. This ensures +// that only a single instance of it is ever running at any one time. +type Processor struct { + mutex sync.Mutex + state uint32 + wait sync.WaitGroup + err *error +} + +// Process will process the given function if first-call, else blocking until +// the first function has returned, returning the same error result. +func (p *Processor) Process(proc Processable) (err error) { + // Acquire state lock. + p.mutex.Lock() + + if p.state != 0 { + // Already running. + // + // Get current err ptr. + errPtr := p.err + + // Wait until finish. + p.mutex.Unlock() + p.wait.Wait() + return *errPtr + } + + // Reset error ptr. + p.err = new(error) + + // Set started. + p.wait.Add(1) + p.state = 1 + p.mutex.Unlock() + + defer func() { + if r := recover(); r != nil { + if err != nil { + rOld := r // wrap the panic so we don't lose existing returned error + r = fmt.Errorf("panic occured after error %q: %v", err.Error(), rOld) + } + + // Catch any panics and wrap as error. + err = fmt.Errorf("caught panic: %v", r) + } + + // Store error. + *p.err = err + + // Mark done. + p.wait.Done() + + // Set stopped. + p.mutex.Lock() + p.state = 0 + p.mutex.Unlock() + }() + + // Run process. + err = proc() + return +} diff --git a/vendor/codeberg.org/gruf/go-runners/run.go b/vendor/codeberg.org/gruf/go-runners/run.go deleted file mode 100644 index 67d19b40c..000000000 --- a/vendor/codeberg.org/gruf/go-runners/run.go +++ /dev/null @@ -1,124 +0,0 @@ -package runners - -import ( - "context" - "errors" - "fmt" - "time" - - "codeberg.org/gruf/go-atomics" -) - -// FuncRunner provides a means of managing long-running functions e.g. main logic loops. -type FuncRunner struct { - // HandOff is the time after which a blocking function will be considered handed off - HandOff time.Duration - - // ErrorHandler is the function that errors are passed to when encountered by the - // provided function. This can be used both for logging, and for error filtering - ErrorHandler func(err error) error - - svc Service // underlying service to manage start/stop - err atomics.Error -} - -// Go will attempt to run 'fn' asynchronously. The provided context is used to propagate requested -// cancel if FuncRunner.Stop() is called. Any returned error will be passed to FuncRunner.ErrorHandler -// for filtering/logging/etc. Any blocking functions will be waited on for FuncRunner.HandOff amount of -// time before considering the function as handed off. Returned bool is success state, i.e. returns true -// if function is successfully handed off or returns within hand off time with nil error. -func (r *FuncRunner) Go(fn func(ctx context.Context) error) bool { - var has bool - - done := make(chan struct{}) - - go func() { - var cancelled bool - - has = r.svc.Run(func(ctx context.Context) { - // reset error - r.err.Store(nil) - - // Run supplied func and set errror if returned - if err := Run(func() error { return fn(ctx) }); err != nil { - r.err.Store(err) - } - - // signal done - close(done) - - // Check if cancelled - select { - case <-ctx.Done(): - cancelled = true - default: - cancelled = false - } - }) - - switch has { - // returned after starting - case true: - // Load set error - err := r.err.Load() - - // filter out errors due FuncRunner.Stop() being called - if cancelled && errors.Is(err, context.Canceled) { - // filter out errors from FuncRunner.Stop() being called - r.err.Store(nil) - } else if err != nil && r.ErrorHandler != nil { - // pass any non-nil error to set handler - r.err.Store(r.ErrorHandler(err)) - } - - // already running - case false: - close(done) - } - }() - - // get valid handoff to use - handoff := r.HandOff - if handoff < 1 { - handoff = time.Second * 5 - } - - select { - // handed off (long-run successful) - case <-time.After(handoff): - return true - - // 'fn' returned, check error - case <-done: - return has - } -} - -// Stop will cancel the context supplied to the running function. -func (r *FuncRunner) Stop() bool { - return r.svc.Stop() -} - -// Err returns the last-set error value. -func (r *FuncRunner) Err() error { - return r.err.Load() -} - -// Run will execute the supplied 'fn' catching any panics. Returns either function-returned error or formatted panic. -func Run(fn func() error) (err error) { - defer func() { - if r := recover(); r != nil { - if e, ok := r.(error); ok { - // wrap and preserve existing error - err = fmt.Errorf("caught panic: %w", e) - } else { - // simply create new error fromt iface - err = fmt.Errorf("caught panic: %v", r) - } - } - }() - - // run supplied func - err = fn() - return -} diff --git a/vendor/codeberg.org/gruf/go-runners/service.go b/vendor/codeberg.org/gruf/go-runners/service.go index c019a10f6..8a7c0051a 100644 --- a/vendor/codeberg.org/gruf/go-runners/service.go +++ b/vendor/codeberg.org/gruf/go-runners/service.go @@ -9,7 +9,7 @@ import ( // changes and preventing multiple instances running. Also providing service state information. type Service struct { state uint32 // 0=stopped, 1=running, 2=stopping - mutex sync.Mutex // mutext protects overall state changes + mutex sync.Mutex // mutex protects overall state changes wait sync.Mutex // wait is used as a single-entity wait-group, only ever locked within 'mutex' ctx chan struct{} // ctx is the current context for running function (or nil if not running) } @@ -62,6 +62,29 @@ func (svc *Service) GoRun(fn func(context.Context)) bool { return true } +// RunWait is functionally the same as .Run(), but blocks until the first instance of .Run() returns. +func (svc *Service) RunWait(fn func(context.Context)) bool { + // Attempt to start the svc + ctx, ok := svc.doStart() + if !ok { + <-ctx // block + return false + } + + defer func() { + // unlock single wait + svc.wait.Unlock() + + // ensure stopped + _ = svc.Stop() + }() + + // Run with context. + fn(CancelCtx(ctx)) + + return true +} + // Stop will attempt to stop the service, cancelling the running function's context. Immediately // returns false if not running, and true only after Service is fully stopped. func (svc *Service) Stop() bool { @@ -108,28 +131,29 @@ func (svc *Service) doStart() (chan struct{}, bool) { // Protect startup svc.mutex.Lock() - if svc.state != 0 /* not stopped */ { - svc.mutex.Unlock() - return nil, false - } - - // state started - svc.state = 1 - if svc.ctx == nil { // this will only have been allocated // if svc.Done() was already called. svc.ctx = make(chan struct{}) } - // Start the waiter - svc.wait.Lock() - // Take our own ptr - // and unlock state ctx := svc.ctx - svc.mutex.Unlock() + if svc.state != 0 { + // State was not stopped. + svc.mutex.Unlock() + return ctx, false + } + + // Set started. + svc.state = 1 + + // Start waiter. + svc.wait.Lock() + + // Unlock and return + svc.mutex.Unlock() return ctx, true } diff --git a/vendor/codeberg.org/gruf/go-sched/scheduler.go b/vendor/codeberg.org/gruf/go-sched/scheduler.go index df19cf18b..537e588fe 100644 --- a/vendor/codeberg.org/gruf/go-sched/scheduler.go +++ b/vendor/codeberg.org/gruf/go-sched/scheduler.go @@ -2,7 +2,6 @@ package sched import ( "context" - "runtime" "sort" "sync" "sync/atomic" @@ -55,11 +54,6 @@ func (sch *Scheduler) Start(gorun func(func())) bool { sch.rgo = func(f func()) { go f() } } - // Set GC finalizer to ensure scheduler stopped - runtime.SetFinalizer(sch, func(sch *Scheduler) { - _ = sch.Stop() - }) - // Unlock start routine block.Unlock() @@ -80,11 +74,16 @@ func (sch *Scheduler) Stop() bool { return sch.svc.Stop() } -// Running will return whether Scheduler is running. +// Running will return whether Scheduler is running (i.e. NOT stopped / stopping). func (sch *Scheduler) Running() bool { return sch.svc.Running() } +// Done returns a channel that's closed when Scheduler.Stop() is called. +func (sch *Scheduler) Done() <-chan struct{} { + return sch.svc.Done() +} + // Schedule will add provided Job to the Scheduler, returning a cancel function. func (sch *Scheduler) Schedule(job *Job) (cancel func()) { switch { @@ -127,20 +126,26 @@ func (sch *Scheduler) Schedule(job *Job) (cancel func()) { // run is the main scheduler run routine, which runs for as long as ctx is valid. func (sch *Scheduler) run(ctx context.Context) { var ( + // now stores the current time, and will only be + // set when the timer channel is set to be the + // 'alwaysticks' channel. this allows minimizing + // the number of calls required to time.Now(). + now time.Time + // timerset represents whether timer was running // for a particular run of the loop. false means - // that tch == neverticks || tch == alwaysticks + // that tch == neverticks || tch == alwaysticks. timerset bool - // timer tick channel (or a never-tick channel) + // timer tick channel (or always / never ticks). tch <-chan time.Time // timer notifies this main routine to wake when - // the job queued needs to be checked for executions + // the job queued needs to be checked for executions. timer *time.Timer // stopdrain will stop and drain the timer - // if it has been running (i.e. timerset == true) + // if it has been running (i.e. timerset == true). stopdrain = func() { if timerset && !timer.Stop() { <-timer.C @@ -148,33 +153,33 @@ func (sch *Scheduler) run(ctx context.Context) { } ) - // Create a stopped timer + // Create a stopped timer. timer = time.NewTimer(1) <-timer.C for { - // Reset timer state + // Reset timer state. timerset = false if len(sch.jobs) > 0 { - // Sort jobs by next occurring - sort.Sort(byNext(sch.jobs)) + // Get now time. + now = time.Now() - // Get execution time - now := time.Now() + // Sort jobs by next occurring. + sort.Sort(byNext(sch.jobs)) - // Get next job time + // Get next job time. next := sch.jobs[0].Next() - // If this job is _just_ about to be ready, we - // don't bother sleeping. It's wasted cycles only - // sleeping for some obscenely tiny amount of time - // we can't guarantee precision for. + // If this job is _just_ about to be ready, we don't bother + // sleeping. It's wasted cycles only sleeping for some obscenely + // tiny amount of time we can't guarantee precision for. if until := next.Sub(now); until <= precision/1e3 { - // This job is behind schedule, set to always tick. + // This job is behind, + // set to always tick. tch = alwaysticks } else { - // Reset timer to period + // Reset timer to period. timer.Reset(until) tch = timer.C timerset = true @@ -191,12 +196,14 @@ func (sch *Scheduler) run(ctx context.Context) { return // Timer ticked, run scheduled - case now := <-tch: + case t := <-tch: if !timerset { - // alwaysticks returns zero times - now = time.Now() + // 'alwaysticks' returns zero + // times, BUT 'now' will have + // been set during above sort. + t = now } - sch.schedule(now) + sch.schedule(t) // Received update, handle job/id case v := <-sch.jch: diff --git a/vendor/github.com/robfig/cron/v3/.gitignore b/vendor/github.com/robfig/cron/v3/.gitignore deleted file mode 100644 index 00268614f..000000000 --- a/vendor/github.com/robfig/cron/v3/.gitignore +++ /dev/null @@ -1,22 +0,0 @@ -# Compiled Object files, Static and Dynamic libs (Shared Objects) -*.o -*.a -*.so - -# Folders -_obj -_test - -# Architecture specific extensions/prefixes -*.[568vq] -[568vq].out - -*.cgo1.go -*.cgo2.c -_cgo_defun.c -_cgo_gotypes.go -_cgo_export.* - -_testmain.go - -*.exe diff --git a/vendor/github.com/robfig/cron/v3/.travis.yml b/vendor/github.com/robfig/cron/v3/.travis.yml deleted file mode 100644 index 4f2ee4d97..000000000 --- a/vendor/github.com/robfig/cron/v3/.travis.yml +++ /dev/null @@ -1 +0,0 @@ -language: go diff --git a/vendor/github.com/robfig/cron/v3/LICENSE b/vendor/github.com/robfig/cron/v3/LICENSE deleted file mode 100644 index 3a0f627ff..000000000 --- a/vendor/github.com/robfig/cron/v3/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -Copyright (C) 2012 Rob Figueiredo -All Rights Reserved. - -MIT LICENSE - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/robfig/cron/v3/README.md b/vendor/github.com/robfig/cron/v3/README.md deleted file mode 100644 index 984c537c0..000000000 --- a/vendor/github.com/robfig/cron/v3/README.md +++ /dev/null @@ -1,125 +0,0 @@ -[](http://godoc.org/github.com/robfig/cron) -[](https://travis-ci.org/robfig/cron) - -# cron - -Cron V3 has been released! - -To download the specific tagged release, run: - - go get github.com/robfig/cron/v3@v3.0.0 - -Import it in your program as: - - import "github.com/robfig/cron/v3" - -It requires Go 1.11 or later due to usage of Go Modules. - -Refer to the documentation here: -http://godoc.org/github.com/robfig/cron - -The rest of this document describes the the advances in v3 and a list of -breaking changes for users that wish to upgrade from an earlier version. - -## Upgrading to v3 (June 2019) - -cron v3 is a major upgrade to the library that addresses all outstanding bugs, -feature requests, and rough edges. It is based on a merge of master which -contains various fixes to issues found over the years and the v2 branch which -contains some backwards-incompatible features like the ability to remove cron -jobs. In addition, v3 adds support for Go Modules, cleans up rough edges like -the timezone support, and fixes a number of bugs. - -New features: - -- Support for Go modules. Callers must now import this library as - `github.com/robfig/cron/v3`, instead of `gopkg.in/...` - -- Fixed bugs: - - 0f01e6b parser: fix combining of Dow and Dom (#70) - - dbf3220 adjust times when rolling the clock forward to handle non-existent midnight (#157) - - eeecf15 spec_test.go: ensure an error is returned on 0 increment (#144) - - 70971dc cron.Entries(): update request for snapshot to include a reply channel (#97) - - 1cba5e6 cron: fix: removing a job causes the next scheduled job to run too late (#206) - -- Standard cron spec parsing by default (first field is "minute"), with an easy - way to opt into the seconds field (quartz-compatible). Although, note that the - year field (optional in Quartz) is not supported. - -- Extensible, key/value logging via an interface that complies with - the https://github.com/go-logr/logr project. - -- The new Chain & JobWrapper types allow you to install "interceptors" to add - cross-cutting behavior like the following: - - Recover any panics from jobs - - Delay a job's execution if the previous run hasn't completed yet - - Skip a job's execution if the previous run hasn't completed yet - - Log each job's invocations - - Notification when jobs are completed - -It is backwards incompatible with both v1 and v2. These updates are required: - -- The v1 branch accepted an optional seconds field at the beginning of the cron - spec. This is non-standard and has led to a lot of confusion. The new default - parser conforms to the standard as described by [the Cron wikipedia page]. - - UPDATING: To retain the old behavior, construct your Cron with a custom - parser: - - // Seconds field, required - cron.New(cron.WithSeconds()) - - // Seconds field, optional - cron.New( - cron.WithParser( - cron.SecondOptional | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor)) - -- The Cron type now accepts functional options on construction rather than the - previous ad-hoc behavior modification mechanisms (setting a field, calling a setter). - - UPDATING: Code that sets Cron.ErrorLogger or calls Cron.SetLocation must be - updated to provide those values on construction. - -- CRON_TZ is now the recommended way to specify the timezone of a single - schedule, which is sanctioned by the specification. The legacy "TZ=" prefix - will continue to be supported since it is unambiguous and easy to do so. - - UPDATING: No update is required. - -- By default, cron will no longer recover panics in jobs that it runs. - Recovering can be surprising (see issue #192) and seems to be at odds with - typical behavior of libraries. Relatedly, the `cron.WithPanicLogger` option - has been removed to accommodate the more general JobWrapper type. - - UPDATING: To opt into panic recovery and configure the panic logger: - - cron.New(cron.WithChain( - cron.Recover(logger), // or use cron.DefaultLogger - )) - -- In adding support for https://github.com/go-logr/logr, `cron.WithVerboseLogger` was - removed, since it is duplicative with the leveled logging. - - UPDATING: Callers should use `WithLogger` and specify a logger that does not - discard `Info` logs. For convenience, one is provided that wraps `*log.Logger`: - - cron.New( - cron.WithLogger(cron.VerbosePrintfLogger(logger))) - - -### Background - Cron spec format - -There are two cron spec formats in common usage: - -- The "standard" cron format, described on [the Cron wikipedia page] and used by - the cron Linux system utility. - -- The cron format used by [the Quartz Scheduler], commonly used for scheduled - jobs in Java software - -[the Cron wikipedia page]: https://en.wikipedia.org/wiki/Cron -[the Quartz Scheduler]: http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/tutorial-lesson-06.html - -The original version of this package included an optional "seconds" field, which -made it incompatible with both of these formats. Now, the "standard" format is -the default format accepted, and the Quartz format is opt-in. diff --git a/vendor/github.com/robfig/cron/v3/chain.go b/vendor/github.com/robfig/cron/v3/chain.go deleted file mode 100644 index 9565b418e..000000000 --- a/vendor/github.com/robfig/cron/v3/chain.go +++ /dev/null @@ -1,92 +0,0 @@ -package cron - -import ( - "fmt" - "runtime" - "sync" - "time" -) - -// JobWrapper decorates the given Job with some behavior. -type JobWrapper func(Job) Job - -// Chain is a sequence of JobWrappers that decorates submitted jobs with -// cross-cutting behaviors like logging or synchronization. -type Chain struct { - wrappers []JobWrapper -} - -// NewChain returns a Chain consisting of the given JobWrappers. -func NewChain(c ...JobWrapper) Chain { - return Chain{c} -} - -// Then decorates the given job with all JobWrappers in the chain. -// -// This: -// NewChain(m1, m2, m3).Then(job) -// is equivalent to: -// m1(m2(m3(job))) -func (c Chain) Then(j Job) Job { - for i := range c.wrappers { - j = c.wrappers[len(c.wrappers)-i-1](j) - } - return j -} - -// Recover panics in wrapped jobs and log them with the provided logger. -func Recover(logger Logger) JobWrapper { - return func(j Job) Job { - return FuncJob(func() { - defer func() { - if r := recover(); r != nil { - const size = 64 << 10 - buf := make([]byte, size) - buf = buf[:runtime.Stack(buf, false)] - err, ok := r.(error) - if !ok { - err = fmt.Errorf("%v", r) - } - logger.Error(err, "panic", "stack", "...\n"+string(buf)) - } - }() - j.Run() - }) - } -} - -// DelayIfStillRunning serializes jobs, delaying subsequent runs until the -// previous one is complete. Jobs running after a delay of more than a minute -// have the delay logged at Info. -func DelayIfStillRunning(logger Logger) JobWrapper { - return func(j Job) Job { - var mu sync.Mutex - return FuncJob(func() { - start := time.Now() - mu.Lock() - defer mu.Unlock() - if dur := time.Since(start); dur > time.Minute { - logger.Info("delay", "duration", dur) - } - j.Run() - }) - } -} - -// SkipIfStillRunning skips an invocation of the Job if a previous invocation is -// still running. It logs skips to the given logger at Info level. -func SkipIfStillRunning(logger Logger) JobWrapper { - return func(j Job) Job { - var ch = make(chan struct{}, 1) - ch <- struct{}{} - return FuncJob(func() { - select { - case v := <-ch: - j.Run() - ch <- v - default: - logger.Info("skip") - } - }) - } -} diff --git a/vendor/github.com/robfig/cron/v3/constantdelay.go b/vendor/github.com/robfig/cron/v3/constantdelay.go deleted file mode 100644 index cd6e7b1be..000000000 --- a/vendor/github.com/robfig/cron/v3/constantdelay.go +++ /dev/null @@ -1,27 +0,0 @@ -package cron - -import "time" - -// ConstantDelaySchedule represents a simple recurring duty cycle, e.g. "Every 5 minutes". -// It does not support jobs more frequent than once a second. -type ConstantDelaySchedule struct { - Delay time.Duration -} - -// Every returns a crontab Schedule that activates once every duration. -// Delays of less than a second are not supported (will round up to 1 second). -// Any fields less than a Second are truncated. -func Every(duration time.Duration) ConstantDelaySchedule { - if duration < time.Second { - duration = time.Second - } - return ConstantDelaySchedule{ - Delay: duration - time.Duration(duration.Nanoseconds())%time.Second, - } -} - -// Next returns the next time this should be run. -// This rounds so that the next activation time will be on the second. -func (schedule ConstantDelaySchedule) Next(t time.Time) time.Time { - return t.Add(schedule.Delay - time.Duration(t.Nanosecond())*time.Nanosecond) -} diff --git a/vendor/github.com/robfig/cron/v3/cron.go b/vendor/github.com/robfig/cron/v3/cron.go deleted file mode 100644 index c7e917665..000000000 --- a/vendor/github.com/robfig/cron/v3/cron.go +++ /dev/null @@ -1,355 +0,0 @@ -package cron - -import ( - "context" - "sort" - "sync" - "time" -) - -// Cron keeps track of any number of entries, invoking the associated func as -// specified by the schedule. It may be started, stopped, and the entries may -// be inspected while running. -type Cron struct { - entries []*Entry - chain Chain - stop chan struct{} - add chan *Entry - remove chan EntryID - snapshot chan chan []Entry - running bool - logger Logger - runningMu sync.Mutex - location *time.Location - parser ScheduleParser - nextID EntryID - jobWaiter sync.WaitGroup -} - -// ScheduleParser is an interface for schedule spec parsers that return a Schedule -type ScheduleParser interface { - Parse(spec string) (Schedule, error) -} - -// Job is an interface for submitted cron jobs. -type Job interface { - Run() -} - -// Schedule describes a job's duty cycle. -type Schedule interface { - // Next returns the next activation time, later than the given time. - // Next is invoked initially, and then each time the job is run. - Next(time.Time) time.Time -} - -// EntryID identifies an entry within a Cron instance -type EntryID int - -// Entry consists of a schedule and the func to execute on that schedule. -type Entry struct { - // ID is the cron-assigned ID of this entry, which may be used to look up a - // snapshot or remove it. - ID EntryID - - // Schedule on which this job should be run. - Schedule Schedule - - // Next time the job will run, or the zero time if Cron has not been - // started or this entry's schedule is unsatisfiable - Next time.Time - - // Prev is the last time this job was run, or the zero time if never. - Prev time.Time - - // WrappedJob is the thing to run when the Schedule is activated. - WrappedJob Job - - // Job is the thing that was submitted to cron. - // It is kept around so that user code that needs to get at the job later, - // e.g. via Entries() can do so. - Job Job -} - -// Valid returns true if this is not the zero entry. -func (e Entry) Valid() bool { return e.ID != 0 } - -// byTime is a wrapper for sorting the entry array by time -// (with zero time at the end). -type byTime []*Entry - -func (s byTime) Len() int { return len(s) } -func (s byTime) Swap(i, j int) { s[i], s[j] = s[j], s[i] } -func (s byTime) Less(i, j int) bool { - // Two zero times should return false. - // Otherwise, zero is "greater" than any other time. - // (To sort it at the end of the list.) - if s[i].Next.IsZero() { - return false - } - if s[j].Next.IsZero() { - return true - } - return s[i].Next.Before(s[j].Next) -} - -// New returns a new Cron job runner, modified by the given options. -// -// Available Settings -// -// Time Zone -// Description: The time zone in which schedules are interpreted -// Default: time.Local -// -// Parser -// Description: Parser converts cron spec strings into cron.Schedules. -// Default: Accepts this spec: https://en.wikipedia.org/wiki/Cron -// -// Chain -// Description: Wrap submitted jobs to customize behavior. -// Default: A chain that recovers panics and logs them to stderr. -// -// See "cron.With*" to modify the default behavior. -func New(opts ...Option) *Cron { - c := &Cron{ - entries: nil, - chain: NewChain(), - add: make(chan *Entry), - stop: make(chan struct{}), - snapshot: make(chan chan []Entry), - remove: make(chan EntryID), - running: false, - runningMu: sync.Mutex{}, - logger: DefaultLogger, - location: time.Local, - parser: standardParser, - } - for _, opt := range opts { - opt(c) - } - return c -} - -// FuncJob is a wrapper that turns a func() into a cron.Job -type FuncJob func() - -func (f FuncJob) Run() { f() } - -// AddFunc adds a func to the Cron to be run on the given schedule. -// The spec is parsed using the time zone of this Cron instance as the default. -// An opaque ID is returned that can be used to later remove it. -func (c *Cron) AddFunc(spec string, cmd func()) (EntryID, error) { - return c.AddJob(spec, FuncJob(cmd)) -} - -// AddJob adds a Job to the Cron to be run on the given schedule. -// The spec is parsed using the time zone of this Cron instance as the default. -// An opaque ID is returned that can be used to later remove it. -func (c *Cron) AddJob(spec string, cmd Job) (EntryID, error) { - schedule, err := c.parser.Parse(spec) - if err != nil { - return 0, err - } - return c.Schedule(schedule, cmd), nil -} - -// Schedule adds a Job to the Cron to be run on the given schedule. -// The job is wrapped with the configured Chain. -func (c *Cron) Schedule(schedule Schedule, cmd Job) EntryID { - c.runningMu.Lock() - defer c.runningMu.Unlock() - c.nextID++ - entry := &Entry{ - ID: c.nextID, - Schedule: schedule, - WrappedJob: c.chain.Then(cmd), - Job: cmd, - } - if !c.running { - c.entries = append(c.entries, entry) - } else { - c.add <- entry - } - return entry.ID -} - -// Entries returns a snapshot of the cron entries. -func (c *Cron) Entries() []Entry { - c.runningMu.Lock() - defer c.runningMu.Unlock() - if c.running { - replyChan := make(chan []Entry, 1) - c.snapshot <- replyChan - return <-replyChan - } - return c.entrySnapshot() -} - -// Location gets the time zone location -func (c *Cron) Location() *time.Location { - return c.location -} - -// Entry returns a snapshot of the given entry, or nil if it couldn't be found. -func (c *Cron) Entry(id EntryID) Entry { - for _, entry := range c.Entries() { - if id == entry.ID { - return entry - } - } - return Entry{} -} - -// Remove an entry from being run in the future. -func (c *Cron) Remove(id EntryID) { - c.runningMu.Lock() - defer c.runningMu.Unlock() - if c.running { - c.remove <- id - } else { - c.removeEntry(id) - } -} - -// Start the cron scheduler in its own goroutine, or no-op if already started. -func (c *Cron) Start() { - c.runningMu.Lock() - defer c.runningMu.Unlock() - if c.running { - return - } - c.running = true - go c.run() -} - -// Run the cron scheduler, or no-op if already running. -func (c *Cron) Run() { - c.runningMu.Lock() - if c.running { - c.runningMu.Unlock() - return - } - c.running = true - c.runningMu.Unlock() - c.run() -} - -// run the scheduler.. this is private just due to the need to synchronize -// access to the 'running' state variable. -func (c *Cron) run() { - c.logger.Info("start") - - // Figure out the next activation times for each entry. - now := c.now() - for _, entry := range c.entries { - entry.Next = entry.Schedule.Next(now) - c.logger.Info("schedule", "now", now, "entry", entry.ID, "next", entry.Next) - } - - for { - // Determine the next entry to run. - sort.Sort(byTime(c.entries)) - - var timer *time.Timer - if len(c.entries) == 0 || c.entries[0].Next.IsZero() { - // If there are no entries yet, just sleep - it still handles new entries - // and stop requests. - timer = time.NewTimer(100000 * time.Hour) - } else { - timer = time.NewTimer(c.entries[0].Next.Sub(now)) - } - - for { - select { - case now = <-timer.C: - now = now.In(c.location) - c.logger.Info("wake", "now", now) - - // Run every entry whose next time was less than now - for _, e := range c.entries { - if e.Next.After(now) || e.Next.IsZero() { - break - } - c.startJob(e.WrappedJob) - e.Prev = e.Next - e.Next = e.Schedule.Next(now) - c.logger.Info("run", "now", now, "entry", e.ID, "next", e.Next) - } - - case newEntry := <-c.add: - timer.Stop() - now = c.now() - newEntry.Next = newEntry.Schedule.Next(now) - c.entries = append(c.entries, newEntry) - c.logger.Info("added", "now", now, "entry", newEntry.ID, "next", newEntry.Next) - - case replyChan := <-c.snapshot: - replyChan <- c.entrySnapshot() - continue - - case <-c.stop: - timer.Stop() - c.logger.Info("stop") - return - - case id := <-c.remove: - timer.Stop() - now = c.now() - c.removeEntry(id) - c.logger.Info("removed", "entry", id) - } - - break - } - } -} - -// startJob runs the given job in a new goroutine. -func (c *Cron) startJob(j Job) { - c.jobWaiter.Add(1) - go func() { - defer c.jobWaiter.Done() - j.Run() - }() -} - -// now returns current time in c location -func (c *Cron) now() time.Time { - return time.Now().In(c.location) -} - -// Stop stops the cron scheduler if it is running; otherwise it does nothing. -// A context is returned so the caller can wait for running jobs to complete. -func (c *Cron) Stop() context.Context { - c.runningMu.Lock() - defer c.runningMu.Unlock() - if c.running { - c.stop <- struct{}{} - c.running = false - } - ctx, cancel := context.WithCancel(context.Background()) - go func() { - c.jobWaiter.Wait() - cancel() - }() - return ctx -} - -// entrySnapshot returns a copy of the current cron entry list. -func (c *Cron) entrySnapshot() []Entry { - var entries = make([]Entry, len(c.entries)) - for i, e := range c.entries { - entries[i] = *e - } - return entries -} - -func (c *Cron) removeEntry(id EntryID) { - var entries []*Entry - for _, e := range c.entries { - if e.ID != id { - entries = append(entries, e) - } - } - c.entries = entries -} diff --git a/vendor/github.com/robfig/cron/v3/doc.go b/vendor/github.com/robfig/cron/v3/doc.go deleted file mode 100644 index fa5d08b4d..000000000 --- a/vendor/github.com/robfig/cron/v3/doc.go +++ /dev/null @@ -1,231 +0,0 @@ -/* -Package cron implements a cron spec parser and job runner. - -Installation - -To download the specific tagged release, run: - - go get github.com/robfig/cron/v3@v3.0.0 - -Import it in your program as: - - import "github.com/robfig/cron/v3" - -It requires Go 1.11 or later due to usage of Go Modules. - -Usage - -Callers may register Funcs to be invoked on a given schedule. Cron will run -them in their own goroutines. - - c := cron.New() - c.AddFunc("30 * * * *", func() { fmt.Println("Every hour on the half hour") }) - c.AddFunc("30 3-6,20-23 * * *", func() { fmt.Println(".. in the range 3-6am, 8-11pm") }) - c.AddFunc("CRON_TZ=Asia/Tokyo 30 04 * * *", func() { fmt.Println("Runs at 04:30 Tokyo time every day") }) - c.AddFunc("@hourly", func() { fmt.Println("Every hour, starting an hour from now") }) - c.AddFunc("@every 1h30m", func() { fmt.Println("Every hour thirty, starting an hour thirty from now") }) - c.Start() - .. - // Funcs are invoked in their own goroutine, asynchronously. - ... - // Funcs may also be added to a running Cron - c.AddFunc("@daily", func() { fmt.Println("Every day") }) - .. - // Inspect the cron job entries' next and previous run times. - inspect(c.Entries()) - .. - c.Stop() // Stop the scheduler (does not stop any jobs already running). - -CRON Expression Format - -A cron expression represents a set of times, using 5 space-separated fields. - - Field name | Mandatory? | Allowed values | Allowed special characters - ---------- | ---------- | -------------- | -------------------------- - Minutes | Yes | 0-59 | * / , - - Hours | Yes | 0-23 | * / , - - Day of month | Yes | 1-31 | * / , - ? - Month | Yes | 1-12 or JAN-DEC | * / , - - Day of week | Yes | 0-6 or SUN-SAT | * / , - ? - -Month and Day-of-week field values are case insensitive. "SUN", "Sun", and -"sun" are equally accepted. - -The specific interpretation of the format is based on the Cron Wikipedia page: -https://en.wikipedia.org/wiki/Cron - -Alternative Formats - -Alternative Cron expression formats support other fields like seconds. You can -implement that by creating a custom Parser as follows. - - cron.New( - cron.WithParser( - cron.NewParser( - cron.SecondOptional | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor))) - -Since adding Seconds is the most common modification to the standard cron spec, -cron provides a builtin function to do that, which is equivalent to the custom -parser you saw earlier, except that its seconds field is REQUIRED: - - cron.New(cron.WithSeconds()) - -That emulates Quartz, the most popular alternative Cron schedule format: -http://www.quartz-scheduler.org/documentation/quartz-2.x/tutorials/crontrigger.html - -Special Characters - -Asterisk ( * ) - -The asterisk indicates that the cron expression will match for all values of the -field; e.g., using an asterisk in the 5th field (month) would indicate every -month. - -Slash ( / ) - -Slashes are used to describe increments of ranges. For example 3-59/15 in the -1st field (minutes) would indicate the 3rd minute of the hour and every 15 -minutes thereafter. The form "*\/..." is equivalent to the form "first-last/...", -that is, an increment over the largest possible range of the field. The form -"N/..." is accepted as meaning "N-MAX/...", that is, starting at N, use the -increment until the end of that specific range. It does not wrap around. - -Comma ( , ) - -Commas are used to separate items of a list. For example, using "MON,WED,FRI" in -the 5th field (day of week) would mean Mondays, Wednesdays and Fridays. - -Hyphen ( - ) - -Hyphens are used to define ranges. For example, 9-17 would indicate every -hour between 9am and 5pm inclusive. - -Question mark ( ? ) - -Question mark may be used instead of '*' for leaving either day-of-month or -day-of-week blank. - -Predefined schedules - -You may use one of several pre-defined schedules in place of a cron expression. - - Entry | Description | Equivalent To - ----- | ----------- | ------------- - @yearly (or @annually) | Run once a year, midnight, Jan. 1st | 0 0 1 1 * - @monthly | Run once a month, midnight, first of month | 0 0 1 * * - @weekly | Run once a week, midnight between Sat/Sun | 0 0 * * 0 - @daily (or @midnight) | Run once a day, midnight | 0 0 * * * - @hourly | Run once an hour, beginning of hour | 0 * * * * - -Intervals - -You may also schedule a job to execute at fixed intervals, starting at the time it's added -or cron is run. This is supported by formatting the cron spec like this: - - @every <duration> - -where "duration" is a string accepted by time.ParseDuration -(http://golang.org/pkg/time/#ParseDuration). - -For example, "@every 1h30m10s" would indicate a schedule that activates after -1 hour, 30 minutes, 10 seconds, and then every interval after that. - -Note: The interval does not take the job runtime into account. For example, -if a job takes 3 minutes to run, and it is scheduled to run every 5 minutes, -it will have only 2 minutes of idle time between each run. - -Time zones - -By default, all interpretation and scheduling is done in the machine's local -time zone (time.Local). You can specify a different time zone on construction: - - cron.New( - cron.WithLocation(time.UTC)) - -Individual cron schedules may also override the time zone they are to be -interpreted in by providing an additional space-separated field at the beginning -of the cron spec, of the form "CRON_TZ=Asia/Tokyo". - -For example: - - # Runs at 6am in time.Local - cron.New().AddFunc("0 6 * * ?", ...) - - # Runs at 6am in America/New_York - nyc, _ := time.LoadLocation("America/New_York") - c := cron.New(cron.WithLocation(nyc)) - c.AddFunc("0 6 * * ?", ...) - - # Runs at 6am in Asia/Tokyo - cron.New().AddFunc("CRON_TZ=Asia/Tokyo 0 6 * * ?", ...) - - # Runs at 6am in Asia/Tokyo - c := cron.New(cron.WithLocation(nyc)) - c.SetLocation("America/New_York") - c.AddFunc("CRON_TZ=Asia/Tokyo 0 6 * * ?", ...) - -The prefix "TZ=(TIME ZONE)" is also supported for legacy compatibility. - -Be aware that jobs scheduled during daylight-savings leap-ahead transitions will -not be run! - -Job Wrappers - -A Cron runner may be configured with a chain of job wrappers to add -cross-cutting functionality to all submitted jobs. For example, they may be used -to achieve the following effects: - - - Recover any panics from jobs (activated by default) - - Delay a job's execution if the previous run hasn't completed yet - - Skip a job's execution if the previous run hasn't completed yet - - Log each job's invocations - -Install wrappers for all jobs added to a cron using the `cron.WithChain` option: - - cron.New(cron.WithChain( - cron.SkipIfStillRunning(logger), - )) - -Install wrappers for individual jobs by explicitly wrapping them: - - job = cron.NewChain( - cron.SkipIfStillRunning(logger), - ).Then(job) - -Thread safety - -Since the Cron service runs concurrently with the calling code, some amount of -care must be taken to ensure proper synchronization. - -All cron methods are designed to be correctly synchronized as long as the caller -ensures that invocations have a clear happens-before ordering between them. - -Logging - -Cron defines a Logger interface that is a subset of the one defined in -github.com/go-logr/logr. It has two logging levels (Info and Error), and -parameters are key/value pairs. This makes it possible for cron logging to plug -into structured logging systems. An adapter, [Verbose]PrintfLogger, is provided -to wrap the standard library *log.Logger. - -For additional insight into Cron operations, verbose logging may be activated -which will record job runs, scheduling decisions, and added or removed jobs. -Activate it with a one-off logger as follows: - - cron.New( - cron.WithLogger( - cron.VerbosePrintfLogger(log.New(os.Stdout, "cron: ", log.LstdFlags)))) - - -Implementation - -Cron entries are stored in an array, sorted by their next activation time. Cron -sleeps until the next job is due to be run. - -Upon waking: - - it runs each entry that is active on that second - - it calculates the next run times for the jobs that were run - - it re-sorts the array of entries by next activation time. - - it goes to sleep until the soonest job. -*/ -package cron diff --git a/vendor/github.com/robfig/cron/v3/logger.go b/vendor/github.com/robfig/cron/v3/logger.go deleted file mode 100644 index b4efcc053..000000000 --- a/vendor/github.com/robfig/cron/v3/logger.go +++ /dev/null @@ -1,86 +0,0 @@ -package cron - -import ( - "io/ioutil" - "log" - "os" - "strings" - "time" -) - -// DefaultLogger is used by Cron if none is specified. -var DefaultLogger Logger = PrintfLogger(log.New(os.Stdout, "cron: ", log.LstdFlags)) - -// DiscardLogger can be used by callers to discard all log messages. -var DiscardLogger Logger = PrintfLogger(log.New(ioutil.Discard, "", 0)) - -// Logger is the interface used in this package for logging, so that any backend -// can be plugged in. It is a subset of the github.com/go-logr/logr interface. -type Logger interface { - // Info logs routine messages about cron's operation. - Info(msg string, keysAndValues ...interface{}) - // Error logs an error condition. - Error(err error, msg string, keysAndValues ...interface{}) -} - -// PrintfLogger wraps a Printf-based logger (such as the standard library "log") -// into an implementation of the Logger interface which logs errors only. -func PrintfLogger(l interface{ Printf(string, ...interface{}) }) Logger { - return printfLogger{l, false} -} - -// VerbosePrintfLogger wraps a Printf-based logger (such as the standard library -// "log") into an implementation of the Logger interface which logs everything. -func VerbosePrintfLogger(l interface{ Printf(string, ...interface{}) }) Logger { - return printfLogger{l, true} -} - -type printfLogger struct { - logger interface{ Printf(string, ...interface{}) } - logInfo bool -} - -func (pl printfLogger) Info(msg string, keysAndValues ...interface{}) { - if pl.logInfo { - keysAndValues = formatTimes(keysAndValues) - pl.logger.Printf( - formatString(len(keysAndValues)), - append([]interface{}{msg}, keysAndValues...)...) - } -} - -func (pl printfLogger) Error(err error, msg string, keysAndValues ...interface{}) { - keysAndValues = formatTimes(keysAndValues) - pl.logger.Printf( - formatString(len(keysAndValues)+2), - append([]interface{}{msg, "error", err}, keysAndValues...)...) -} - -// formatString returns a logfmt-like format string for the number of -// key/values. -func formatString(numKeysAndValues int) string { - var sb strings.Builder - sb.WriteString("%s") - if numKeysAndValues > 0 { - sb.WriteString(", ") - } - for i := 0; i < numKeysAndValues/2; i++ { - if i > 0 { - sb.WriteString(", ") - } - sb.WriteString("%v=%v") - } - return sb.String() -} - -// formatTimes formats any time.Time values as RFC3339. -func formatTimes(keysAndValues []interface{}) []interface{} { - var formattedArgs []interface{} - for _, arg := range keysAndValues { - if t, ok := arg.(time.Time); ok { - arg = t.Format(time.RFC3339) - } - formattedArgs = append(formattedArgs, arg) - } - return formattedArgs -} diff --git a/vendor/github.com/robfig/cron/v3/option.go b/vendor/github.com/robfig/cron/v3/option.go deleted file mode 100644 index 09e4278e7..000000000 --- a/vendor/github.com/robfig/cron/v3/option.go +++ /dev/null @@ -1,45 +0,0 @@ -package cron - -import ( - "time" -) - -// Option represents a modification to the default behavior of a Cron. -type Option func(*Cron) - -// WithLocation overrides the timezone of the cron instance. -func WithLocation(loc *time.Location) Option { - return func(c *Cron) { - c.location = loc - } -} - -// WithSeconds overrides the parser used for interpreting job schedules to -// include a seconds field as the first one. -func WithSeconds() Option { - return WithParser(NewParser( - Second | Minute | Hour | Dom | Month | Dow | Descriptor, - )) -} - -// WithParser overrides the parser used for interpreting job schedules. -func WithParser(p ScheduleParser) Option { - return func(c *Cron) { - c.parser = p - } -} - -// WithChain specifies Job wrappers to apply to all jobs added to this cron. -// Refer to the Chain* functions in this package for provided wrappers. -func WithChain(wrappers ...JobWrapper) Option { - return func(c *Cron) { - c.chain = NewChain(wrappers...) - } -} - -// WithLogger uses the provided logger. -func WithLogger(logger Logger) Option { - return func(c *Cron) { - c.logger = logger - } -} diff --git a/vendor/github.com/robfig/cron/v3/parser.go b/vendor/github.com/robfig/cron/v3/parser.go deleted file mode 100644 index 3cf8879f7..000000000 --- a/vendor/github.com/robfig/cron/v3/parser.go +++ /dev/null @@ -1,434 +0,0 @@ -package cron - -import ( - "fmt" - "math" - "strconv" - "strings" - "time" -) - -// Configuration options for creating a parser. Most options specify which -// fields should be included, while others enable features. If a field is not -// included the parser will assume a default value. These options do not change -// the order fields are parse in. -type ParseOption int - -const ( - Second ParseOption = 1 << iota // Seconds field, default 0 - SecondOptional // Optional seconds field, default 0 - Minute // Minutes field, default 0 - Hour // Hours field, default 0 - Dom // Day of month field, default * - Month // Month field, default * - Dow // Day of week field, default * - DowOptional // Optional day of week field, default * - Descriptor // Allow descriptors such as @monthly, @weekly, etc. -) - -var places = []ParseOption{ - Second, - Minute, - Hour, - Dom, - Month, - Dow, -} - -var defaults = []string{ - "0", - "0", - "0", - "*", - "*", - "*", -} - -// A custom Parser that can be configured. -type Parser struct { - options ParseOption -} - -// NewParser creates a Parser with custom options. -// -// It panics if more than one Optional is given, since it would be impossible to -// correctly infer which optional is provided or missing in general. -// -// Examples -// -// // Standard parser without descriptors -// specParser := NewParser(Minute | Hour | Dom | Month | Dow) -// sched, err := specParser.Parse("0 0 15 */3 *") -// -// // Same as above, just excludes time fields -// subsParser := NewParser(Dom | Month | Dow) -// sched, err := specParser.Parse("15 */3 *") -// -// // Same as above, just makes Dow optional -// subsParser := NewParser(Dom | Month | DowOptional) -// sched, err := specParser.Parse("15 */3") -// -func NewParser(options ParseOption) Parser { - optionals := 0 - if options&DowOptional > 0 { - optionals++ - } - if options&SecondOptional > 0 { - optionals++ - } - if optionals > 1 { - panic("multiple optionals may not be configured") - } - return Parser{options} -} - -// Parse returns a new crontab schedule representing the given spec. -// It returns a descriptive error if the spec is not valid. -// It accepts crontab specs and features configured by NewParser. -func (p Parser) Parse(spec string) (Schedule, error) { - if len(spec) == 0 { - return nil, fmt.Errorf("empty spec string") - } - - // Extract timezone if present - var loc = time.Local - if strings.HasPrefix(spec, "TZ=") || strings.HasPrefix(spec, "CRON_TZ=") { - var err error - i := strings.Index(spec, " ") - eq := strings.Index(spec, "=") - if loc, err = time.LoadLocation(spec[eq+1 : i]); err != nil { - return nil, fmt.Errorf("provided bad location %s: %v", spec[eq+1:i], err) - } - spec = strings.TrimSpace(spec[i:]) - } - - // Handle named schedules (descriptors), if configured - if strings.HasPrefix(spec, "@") { - if p.options&Descriptor == 0 { - return nil, fmt.Errorf("parser does not accept descriptors: %v", spec) - } - return parseDescriptor(spec, loc) - } - - // Split on whitespace. - fields := strings.Fields(spec) - - // Validate & fill in any omitted or optional fields - var err error - fields, err = normalizeFields(fields, p.options) - if err != nil { - return nil, err - } - - field := func(field string, r bounds) uint64 { - if err != nil { - return 0 - } - var bits uint64 - bits, err = getField(field, r) - return bits - } - - var ( - second = field(fields[0], seconds) - minute = field(fields[1], minutes) - hour = field(fields[2], hours) - dayofmonth = field(fields[3], dom) - month = field(fields[4], months) - dayofweek = field(fields[5], dow) - ) - if err != nil { - return nil, err - } - - return &SpecSchedule{ - Second: second, - Minute: minute, - Hour: hour, - Dom: dayofmonth, - Month: month, - Dow: dayofweek, - Location: loc, - }, nil -} - -// normalizeFields takes a subset set of the time fields and returns the full set -// with defaults (zeroes) populated for unset fields. -// -// As part of performing this function, it also validates that the provided -// fields are compatible with the configured options. -func normalizeFields(fields []string, options ParseOption) ([]string, error) { - // Validate optionals & add their field to options - optionals := 0 - if options&SecondOptional > 0 { - options |= Second - optionals++ - } - if options&DowOptional > 0 { - options |= Dow - optionals++ - } - if optionals > 1 { - return nil, fmt.Errorf("multiple optionals may not be configured") - } - - // Figure out how many fields we need - max := 0 - for _, place := range places { - if options&place > 0 { - max++ - } - } - min := max - optionals - - // Validate number of fields - if count := len(fields); count < min || count > max { - if min == max { - return nil, fmt.Errorf("expected exactly %d fields, found %d: %s", min, count, fields) - } - return nil, fmt.Errorf("expected %d to %d fields, found %d: %s", min, max, count, fields) - } - - // Populate the optional field if not provided - if min < max && len(fields) == min { - switch { - case options&DowOptional > 0: - fields = append(fields, defaults[5]) // TODO: improve access to default - case options&SecondOptional > 0: - fields = append([]string{defaults[0]}, fields...) - default: - return nil, fmt.Errorf("unknown optional field") - } - } - - // Populate all fields not part of options with their defaults - n := 0 - expandedFields := make([]string, len(places)) - copy(expandedFields, defaults) - for i, place := range places { - if options&place > 0 { - expandedFields[i] = fields[n] - n++ - } - } - return expandedFields, nil -} - -var standardParser = NewParser( - Minute | Hour | Dom | Month | Dow | Descriptor, -) - -// ParseStandard returns a new crontab schedule representing the given -// standardSpec (https://en.wikipedia.org/wiki/Cron). It requires 5 entries -// representing: minute, hour, day of month, month and day of week, in that -// order. It returns a descriptive error if the spec is not valid. -// -// It accepts -// - Standard crontab specs, e.g. "* * * * ?" -// - Descriptors, e.g. "@midnight", "@every 1h30m" -func ParseStandard(standardSpec string) (Schedule, error) { - return standardParser.Parse(standardSpec) -} - -// getField returns an Int with the bits set representing all of the times that -// the field represents or error parsing field value. A "field" is a comma-separated -// list of "ranges". -func getField(field string, r bounds) (uint64, error) { - var bits uint64 - ranges := strings.FieldsFunc(field, func(r rune) bool { return r == ',' }) - for _, expr := range ranges { - bit, err := getRange(expr, r) - if err != nil { - return bits, err - } - bits |= bit - } - return bits, nil -} - -// getRange returns the bits indicated by the given expression: -// number | number "-" number [ "/" number ] -// or error parsing range. -func getRange(expr string, r bounds) (uint64, error) { - var ( - start, end, step uint - rangeAndStep = strings.Split(expr, "/") - lowAndHigh = strings.Split(rangeAndStep[0], "-") - singleDigit = len(lowAndHigh) == 1 - err error - ) - - var extra uint64 - if lowAndHigh[0] == "*" || lowAndHigh[0] == "?" { - start = r.min - end = r.max - extra = starBit - } else { - start, err = parseIntOrName(lowAndHigh[0], r.names) - if err != nil { - return 0, err - } - switch len(lowAndHigh) { - case 1: - end = start - case 2: - end, err = parseIntOrName(lowAndHigh[1], r.names) - if err != nil { - return 0, err - } - default: - return 0, fmt.Errorf("too many hyphens: %s", expr) - } - } - - switch len(rangeAndStep) { - case 1: - step = 1 - case 2: - step, err = mustParseInt(rangeAndStep[1]) - if err != nil { - return 0, err - } - - // Special handling: "N/step" means "N-max/step". - if singleDigit { - end = r.max - } - if step > 1 { - extra = 0 - } - default: - return 0, fmt.Errorf("too many slashes: %s", expr) - } - - if start < r.min { - return 0, fmt.Errorf("beginning of range (%d) below minimum (%d): %s", start, r.min, expr) - } - if end > r.max { - return 0, fmt.Errorf("end of range (%d) above maximum (%d): %s", end, r.max, expr) - } - if start > end { - return 0, fmt.Errorf("beginning of range (%d) beyond end of range (%d): %s", start, end, expr) - } - if step == 0 { - return 0, fmt.Errorf("step of range should be a positive number: %s", expr) - } - - return getBits(start, end, step) | extra, nil -} - -// parseIntOrName returns the (possibly-named) integer contained in expr. -func parseIntOrName(expr string, names map[string]uint) (uint, error) { - if names != nil { - if namedInt, ok := names[strings.ToLower(expr)]; ok { - return namedInt, nil - } - } - return mustParseInt(expr) -} - -// mustParseInt parses the given expression as an int or returns an error. -func mustParseInt(expr string) (uint, error) { - num, err := strconv.Atoi(expr) - if err != nil { - return 0, fmt.Errorf("failed to parse int from %s: %s", expr, err) - } - if num < 0 { - return 0, fmt.Errorf("negative number (%d) not allowed: %s", num, expr) - } - - return uint(num), nil -} - -// getBits sets all bits in the range [min, max], modulo the given step size. -func getBits(min, max, step uint) uint64 { - var bits uint64 - - // If step is 1, use shifts. - if step == 1 { - return ^(math.MaxUint64 << (max + 1)) & (math.MaxUint64 << min) - } - - // Else, use a simple loop. - for i := min; i <= max; i += step { - bits |= 1 << i - } - return bits -} - -// all returns all bits within the given bounds. (plus the star bit) -func all(r bounds) uint64 { - return getBits(r.min, r.max, 1) | starBit -} - -// parseDescriptor returns a predefined schedule for the expression, or error if none matches. -func parseDescriptor(descriptor string, loc *time.Location) (Schedule, error) { - switch descriptor { - case "@yearly", "@annually": - return &SpecSchedule{ - Second: 1 << seconds.min, - Minute: 1 << minutes.min, - Hour: 1 << hours.min, - Dom: 1 << dom.min, - Month: 1 << months.min, - Dow: all(dow), - Location: loc, - }, nil - - case "@monthly": - return &SpecSchedule{ - Second: 1 << seconds.min, - Minute: 1 << minutes.min, - Hour: 1 << hours.min, - Dom: 1 << dom.min, - Month: all(months), - Dow: all(dow), - Location: loc, - }, nil - - case "@weekly": - return &SpecSchedule{ - Second: 1 << seconds.min, - Minute: 1 << minutes.min, - Hour: 1 << hours.min, - Dom: all(dom), - Month: all(months), - Dow: 1 << dow.min, - Location: loc, - }, nil - - case "@daily", "@midnight": - return &SpecSchedule{ - Second: 1 << seconds.min, - Minute: 1 << minutes.min, - Hour: 1 << hours.min, - Dom: all(dom), - Month: all(months), - Dow: all(dow), - Location: loc, - }, nil - - case "@hourly": - return &SpecSchedule{ - Second: 1 << seconds.min, - Minute: 1 << minutes.min, - Hour: all(hours), - Dom: all(dom), - Month: all(months), - Dow: all(dow), - Location: loc, - }, nil - - } - - const every = "@every " - if strings.HasPrefix(descriptor, every) { - duration, err := time.ParseDuration(descriptor[len(every):]) - if err != nil { - return nil, fmt.Errorf("failed to parse duration %s: %s", descriptor, err) - } - return Every(duration), nil - } - - return nil, fmt.Errorf("unrecognized descriptor: %s", descriptor) -} diff --git a/vendor/github.com/robfig/cron/v3/spec.go b/vendor/github.com/robfig/cron/v3/spec.go deleted file mode 100644 index fa1e241e5..000000000 --- a/vendor/github.com/robfig/cron/v3/spec.go +++ /dev/null @@ -1,188 +0,0 @@ -package cron - -import "time" - -// SpecSchedule specifies a duty cycle (to the second granularity), based on a -// traditional crontab specification. It is computed initially and stored as bit sets. -type SpecSchedule struct { - Second, Minute, Hour, Dom, Month, Dow uint64 - - // Override location for this schedule. - Location *time.Location -} - -// bounds provides a range of acceptable values (plus a map of name to value). -type bounds struct { - min, max uint - names map[string]uint -} - -// The bounds for each field. -var ( - seconds = bounds{0, 59, nil} - minutes = bounds{0, 59, nil} - hours = bounds{0, 23, nil} - dom = bounds{1, 31, nil} - months = bounds{1, 12, map[string]uint{ - "jan": 1, - "feb": 2, - "mar": 3, - "apr": 4, - "may": 5, - "jun": 6, - "jul": 7, - "aug": 8, - "sep": 9, - "oct": 10, - "nov": 11, - "dec": 12, - }} - dow = bounds{0, 6, map[string]uint{ - "sun": 0, - "mon": 1, - "tue": 2, - "wed": 3, - "thu": 4, - "fri": 5, - "sat": 6, - }} -) - -const ( - // Set the top bit if a star was included in the expression. - starBit = 1 << 63 -) - -// Next returns the next time this schedule is activated, greater than the given -// time. If no time can be found to satisfy the schedule, return the zero time. -func (s *SpecSchedule) Next(t time.Time) time.Time { - // General approach - // - // For Month, Day, Hour, Minute, Second: - // Check if the time value matches. If yes, continue to the next field. - // If the field doesn't match the schedule, then increment the field until it matches. - // While incrementing the field, a wrap-around brings it back to the beginning - // of the field list (since it is necessary to re-verify previous field - // values) - - // Convert the given time into the schedule's timezone, if one is specified. - // Save the original timezone so we can convert back after we find a time. - // Note that schedules without a time zone specified (time.Local) are treated - // as local to the time provided. - origLocation := t.Location() - loc := s.Location - if loc == time.Local { - loc = t.Location() - } - if s.Location != time.Local { - t = t.In(s.Location) - } - - // Start at the earliest possible time (the upcoming second). - t = t.Add(1*time.Second - time.Duration(t.Nanosecond())*time.Nanosecond) - - // This flag indicates whether a field has been incremented. - added := false - - // If no time is found within five years, return zero. - yearLimit := t.Year() + 5 - -WRAP: - if t.Year() > yearLimit { - return time.Time{} - } - - // Find the first applicable month. - // If it's this month, then do nothing. - for 1<<uint(t.Month())&s.Month == 0 { - // If we have to add a month, reset the other parts to 0. - if !added { - added = true - // Otherwise, set the date at the beginning (since the current time is irrelevant). - t = time.Date(t.Year(), t.Month(), 1, 0, 0, 0, 0, loc) - } - t = t.AddDate(0, 1, 0) - - // Wrapped around. - if t.Month() == time.January { - goto WRAP - } - } - - // Now get a day in that month. - // - // NOTE: This causes issues for daylight savings regimes where midnight does - // not exist. For example: Sao Paulo has DST that transforms midnight on - // 11/3 into 1am. Handle that by noticing when the Hour ends up != 0. - for !dayMatches(s, t) { - if !added { - added = true - t = time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, loc) - } - t = t.AddDate(0, 0, 1) - // Notice if the hour is no longer midnight due to DST. - // Add an hour if it's 23, subtract an hour if it's 1. - if t.Hour() != 0 { - if t.Hour() > 12 { - t = t.Add(time.Duration(24-t.Hour()) * time.Hour) - } else { - t = t.Add(time.Duration(-t.Hour()) * time.Hour) - } - } - - if t.Day() == 1 { - goto WRAP - } - } - - for 1<<uint(t.Hour())&s.Hour == 0 { - if !added { - added = true - t = time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), 0, 0, 0, loc) - } - t = t.Add(1 * time.Hour) - - if t.Hour() == 0 { - goto WRAP - } - } - - for 1<<uint(t.Minute())&s.Minute == 0 { - if !added { - added = true - t = t.Truncate(time.Minute) - } - t = t.Add(1 * time.Minute) - - if t.Minute() == 0 { - goto WRAP - } - } - - for 1<<uint(t.Second())&s.Second == 0 { - if !added { - added = true - t = t.Truncate(time.Second) - } - t = t.Add(1 * time.Second) - - if t.Second() == 0 { - goto WRAP - } - } - - return t.In(origLocation) -} - -// dayMatches returns true if the schedule's day-of-week and day-of-month -// restrictions are satisfied by the given time. -func dayMatches(s *SpecSchedule, t time.Time) bool { - var ( - domMatch bool = 1<<uint(t.Day())&s.Dom > 0 - dowMatch bool = 1<<uint(t.Weekday())&s.Dow > 0 - ) - if s.Dom&starBit > 0 || s.Dow&starBit > 0 { - return domMatch && dowMatch - } - return domMatch || dowMatch -} diff --git a/vendor/modules.txt b/vendor/modules.txt index a4d6fee9c..fbb4cab40 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,8 +1,8 @@ # codeberg.org/gruf/go-atomics v1.1.0 ## explicit; go 1.16 codeberg.org/gruf/go-atomics -# codeberg.org/gruf/go-bitutil v1.0.1 -## explicit; go 1.16 +# codeberg.org/gruf/go-bitutil v1.1.0 +## explicit; go 1.19 codeberg.org/gruf/go-bitutil # codeberg.org/gruf/go-bytes v1.0.2 ## explicit; go 1.14 @@ -18,7 +18,7 @@ codeberg.org/gruf/go-byteutil codeberg.org/gruf/go-cache/v3 codeberg.org/gruf/go-cache/v3/result codeberg.org/gruf/go-cache/v3/ttl -# codeberg.org/gruf/go-debug v1.2.0 +# codeberg.org/gruf/go-debug v1.3.0 ## explicit; go 1.16 codeberg.org/gruf/go-debug # codeberg.org/gruf/go-errors/v2 v2.1.1 @@ -58,10 +58,10 @@ codeberg.org/gruf/go-mutexes # codeberg.org/gruf/go-pools v1.1.0 ## explicit; go 1.16 codeberg.org/gruf/go-pools -# codeberg.org/gruf/go-runners v1.5.1 -## explicit; go 1.14 +# codeberg.org/gruf/go-runners v1.6.0 +## explicit; go 1.19 codeberg.org/gruf/go-runners -# codeberg.org/gruf/go-sched v1.2.0 +# codeberg.org/gruf/go-sched v1.2.3 ## explicit; go 1.19 codeberg.org/gruf/go-sched # codeberg.org/gruf/go-store/v2 v2.2.1 @@ -379,9 +379,6 @@ github.com/quasoft/memstore # github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 ## explicit; go 1.12 github.com/remyoudompheng/bigfft -# github.com/robfig/cron/v3 v3.0.1 -## explicit; go 1.12 -github.com/robfig/cron/v3 # github.com/rs/xid v1.4.0 ## explicit; go 1.12 github.com/rs/xid |