From 87cff71af95d2cef095a5feea40e48b40576b3d0 Mon Sep 17 00:00:00 2001
From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com>
Date: Tue, 30 Jul 2024 11:58:31 +0000
Subject: [feature] persist worker queues to db (#3042)
* persist queued worker tasks to database on shutdown, fill worker queues from database on startup
* ensure the tasks are sorted by creation time before pushing them
* add migration to insert WorkerTask{} into database, add test for worker task persistence
* add test for recovering worker queues from database
* quick tweak
* whoops we ended up with double cleaner job scheduling
* insert each task separately, because bun is throwing some reflection error??
* add specific checking of cancelled worker contexts
* add http request signing to deliveries recovered from database
* add test for outgoing public key ID being correctly set on delivery
* replace select with Queue.PopCtx()
* get rid of loop now we don't use it
* remove field now we don't use it
* ensure that signing func is set
* header values weren't being copied over :facepalm:
* use ptr for httpclient.Request in delivery
* move worker queue filling to later in server init process
* fix rebase issues
* make logging less shouty
* use slices.Delete() instead of copying / reslicing
* have database return tasks in ascending order instead of sorting them
* add a 1 minute timeout to persisting worker queues
---
internal/db/bundb/bundb.go | 4 ++
.../20240617134210_add_worker_tasks_table.go | 51 +++++++++++++++++++
internal/db/bundb/workertask.go | 58 ++++++++++++++++++++++
internal/db/db.go | 1 +
internal/db/workertask.go | 35 +++++++++++++
5 files changed, 149 insertions(+)
create mode 100644 internal/db/bundb/migrations/20240617134210_add_worker_tasks_table.go
create mode 100644 internal/db/bundb/workertask.go
create mode 100644 internal/db/workertask.go
(limited to 'internal/db')
diff --git a/internal/db/bundb/bundb.go b/internal/db/bundb/bundb.go
index 070d4eb91..d5071d141 100644
--- a/internal/db/bundb/bundb.go
+++ b/internal/db/bundb/bundb.go
@@ -84,6 +84,7 @@ type DBService struct {
db.Timeline
db.User
db.Tombstone
+ db.WorkerTask
db *bun.DB
}
@@ -302,6 +303,9 @@ func NewBunDBService(ctx context.Context, state *state.State) (db.DB, error) {
db: db,
state: state,
},
+ WorkerTask: &workerTaskDB{
+ db: db,
+ },
db: db,
}
diff --git a/internal/db/bundb/migrations/20240617134210_add_worker_tasks_table.go b/internal/db/bundb/migrations/20240617134210_add_worker_tasks_table.go
new file mode 100644
index 000000000..3b0ebcfd8
--- /dev/null
+++ b/internal/db/bundb/migrations/20240617134210_add_worker_tasks_table.go
@@ -0,0 +1,51 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package migrations
+
+import (
+ "context"
+
+ "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+ "github.com/uptrace/bun"
+)
+
+func init() {
+ up := func(ctx context.Context, db *bun.DB) error {
+ return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+ // WorkerTask table.
+ if _, err := tx.
+ NewCreateTable().
+ Model(>smodel.WorkerTask{}).
+ IfNotExists().
+ Exec(ctx); err != nil {
+ return err
+ }
+ return nil
+ })
+ }
+
+ down := func(ctx context.Context, db *bun.DB) error {
+ return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+ return nil
+ })
+ }
+
+ if err := Migrations.Register(up, down); err != nil {
+ panic(err)
+ }
+}
diff --git a/internal/db/bundb/workertask.go b/internal/db/bundb/workertask.go
new file mode 100644
index 000000000..eec51530d
--- /dev/null
+++ b/internal/db/bundb/workertask.go
@@ -0,0 +1,58 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package bundb
+
+import (
+ "context"
+ "errors"
+
+ "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+ "github.com/uptrace/bun"
+)
+
+type workerTaskDB struct{ db *bun.DB }
+
+func (w *workerTaskDB) GetWorkerTasks(ctx context.Context) ([]*gtsmodel.WorkerTask, error) {
+ var tasks []*gtsmodel.WorkerTask
+ if err := w.db.NewSelect().
+ Model(&tasks).
+ OrderExpr("? ASC", bun.Ident("created_at")).
+ Scan(ctx); err != nil {
+ return nil, err
+ }
+ return tasks, nil
+}
+
+func (w *workerTaskDB) PutWorkerTasks(ctx context.Context, tasks []*gtsmodel.WorkerTask) error {
+ var errs []error
+ for _, task := range tasks {
+ _, err := w.db.NewInsert().Model(task).Exec(ctx)
+ if err != nil {
+ errs = append(errs, err)
+ }
+ }
+ return errors.Join(errs...)
+}
+
+func (w *workerTaskDB) DeleteWorkerTaskByID(ctx context.Context, id uint) error {
+ _, err := w.db.NewDelete().
+ Table("worker_tasks").
+ Where("? = ?", bun.Ident("id"), id).
+ Exec(ctx)
+ return err
+}
diff --git a/internal/db/db.go b/internal/db/db.go
index 4b2152732..cd621871a 100644
--- a/internal/db/db.go
+++ b/internal/db/db.go
@@ -56,4 +56,5 @@ type DB interface {
Timeline
User
Tombstone
+ WorkerTask
}
diff --git a/internal/db/workertask.go b/internal/db/workertask.go
new file mode 100644
index 000000000..0276f231a
--- /dev/null
+++ b/internal/db/workertask.go
@@ -0,0 +1,35 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package db
+
+import (
+ "context"
+
+ "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+type WorkerTask interface {
+ // GetWorkerTasks fetches all persisted worker tasks from the database.
+ GetWorkerTasks(ctx context.Context) ([]*gtsmodel.WorkerTask, error)
+
+ // PutWorkerTasks persists the given worker tasks to the database.
+ PutWorkerTasks(ctx context.Context, tasks []*gtsmodel.WorkerTask) error
+
+ // DeleteWorkerTask deletes worker task with given ID from database.
+ DeleteWorkerTaskByID(ctx context.Context, id uint) error
+}
--
cgit v1.2.3