feat: initial tracing support (#1623)

7 files changed, 335 insertions, 3 deletions

diff --git a/internal/config/config.go b/internal/config/config.go
index a1570bbaf..b5c228a35 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -126,6 +126,11 @@ type Configuration struct {
 	OIDCLinkExisting     bool     `name:"oidc-link-existing" usage:"link existing user accounts to OIDC logins based on the stored email value"`
 	OIDCAdminGroups      []string `name:"oidc-admin-groups" usage:"Membership of one of the listed groups makes someone a GtS admin"`
 
+	TracingEnabled           bool   `name:"tracing-enabled" usage:"Enable OTLP Tracing"`
+	TracingTransport         string `name:"tracing-transport" usage:"grpc or jaeger"`
+	TracingEndpoint          string `name:"tracing-endpoint" usage:"Endpoint of your trace collector. Eg., 'localhost:4317' for gRPC, 'http://localhost:14268/api/traces' for jaeger"`
+	TracingInsecureTransport bool   `name:"tracing-insecure" usage:"Disable HTTPS for the gRPC transport protocol"`
+
 	SMTPHost               string `name:"smtp-host" usage:"Host of the smtp server. Eg., 'smtp.eu.mailgun.org'"`
 	SMTPPort               int    `name:"smtp-port" usage:"Port of the smtp server. Eg., 587"`
 	SMTPUsername           string `name:"smtp-username" usage:"Username to authenticate with the smtp server as. Eg., 'postmaster@mail.example.org'"`
@@ -153,7 +158,7 @@ type Configuration struct {
 	AdminTransPath        string `name:"path" usage:"the path of the file to import from/export to"`
 	AdminMediaPruneDryRun bool   `name:"dry-run" usage:"perform a dry run and only log number of items eligible for pruning"`
 
-	RequestIDHeader string `name:"request-id-header" usage:"Header to extract the Request ID from. Eg.,'X-Request-Id'"`
+	RequestIDHeader string `name:"request-id-header" usage:"Header to extract the Request ID from. Eg.,'X-Request-Id'."`
 }
 
 type CacheConfiguration struct {
diff --git a/internal/config/defaults.go b/internal/config/defaults.go
index 1dc446e4c..54672c159 100644
--- a/internal/config/defaults.go
+++ b/internal/config/defaults.go
@@ -109,6 +109,11 @@ var Defaults = Configuration{
 	SMTPFrom:               "GoToSocial",
 	SMTPDiscloseRecipients: false,
 
+	TracingEnabled:           false,
+	TracingTransport:         "grpc",
+	TracingEndpoint:          "",
+	TracingInsecureTransport: false,
+
 	SyslogEnabled:  false,
 	SyslogProtocol: "udp",
 	SyslogAddress:  "localhost:514",
diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go
index 236d1ea36..b635f7b8e 100644
--- a/internal/config/helpers.gen.go
+++ b/internal/config/helpers.gen.go
@@ -1799,6 +1799,106 @@ func GetOIDCAdminGroups() []string { return global.GetOIDCAdminGroups() }
 // SetOIDCAdminGroups safely sets the value for global configuration 'OIDCAdminGroups' field
 func SetOIDCAdminGroups(v []string) { global.SetOIDCAdminGroups(v) }
 
+// GetTracingEnabled safely fetches the Configuration value for state's 'TracingEnabled' field
+func (st *ConfigState) GetTracingEnabled() (v bool) {
+	st.mutex.Lock()
+	v = st.config.TracingEnabled
+	st.mutex.Unlock()
+	return
+}
+
+// SetTracingEnabled safely sets the Configuration value for state's 'TracingEnabled' field
+func (st *ConfigState) SetTracingEnabled(v bool) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.TracingEnabled = v
+	st.reloadToViper()
+}
+
+// TracingEnabledFlag returns the flag name for the 'TracingEnabled' field
+func TracingEnabledFlag() string { return "tracing-enabled" }
+
+// GetTracingEnabled safely fetches the value for global configuration 'TracingEnabled' field
+func GetTracingEnabled() bool { return global.GetTracingEnabled() }
+
+// SetTracingEnabled safely sets the value for global configuration 'TracingEnabled' field
+func SetTracingEnabled(v bool) { global.SetTracingEnabled(v) }
+
+// GetTracingTransport safely fetches the Configuration value for state's 'TracingTransport' field
+func (st *ConfigState) GetTracingTransport() (v string) {
+	st.mutex.Lock()
+	v = st.config.TracingTransport
+	st.mutex.Unlock()
+	return
+}
+
+// SetTracingTransport safely sets the Configuration value for state's 'TracingTransport' field
+func (st *ConfigState) SetTracingTransport(v string) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.TracingTransport = v
+	st.reloadToViper()
+}
+
+// TracingTransportFlag returns the flag name for the 'TracingTransport' field
+func TracingTransportFlag() string { return "tracing-transport" }
+
+// GetTracingTransport safely fetches the value for global configuration 'TracingTransport' field
+func GetTracingTransport() string { return global.GetTracingTransport() }
+
+// SetTracingTransport safely sets the value for global configuration 'TracingTransport' field
+func SetTracingTransport(v string) { global.SetTracingTransport(v) }
+
+// GetTracingEndpoint safely fetches the Configuration value for state's 'TracingEndpoint' field
+func (st *ConfigState) GetTracingEndpoint() (v string) {
+	st.mutex.Lock()
+	v = st.config.TracingEndpoint
+	st.mutex.Unlock()
+	return
+}
+
+// SetTracingEndpoint safely sets the Configuration value for state's 'TracingEndpoint' field
+func (st *ConfigState) SetTracingEndpoint(v string) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.TracingEndpoint = v
+	st.reloadToViper()
+}
+
+// TracingEndpointFlag returns the flag name for the 'TracingEndpoint' field
+func TracingEndpointFlag() string { return "tracing-endpoint" }
+
+// GetTracingEndpoint safely fetches the value for global configuration 'TracingEndpoint' field
+func GetTracingEndpoint() string { return global.GetTracingEndpoint() }
+
+// SetTracingEndpoint safely sets the value for global configuration 'TracingEndpoint' field
+func SetTracingEndpoint(v string) { global.SetTracingEndpoint(v) }
+
+// GetTracingInsecureTransport safely fetches the Configuration value for state's 'TracingInsecureTransport' field
+func (st *ConfigState) GetTracingInsecureTransport() (v bool) {
+	st.mutex.Lock()
+	v = st.config.TracingInsecureTransport
+	st.mutex.Unlock()
+	return
+}
+
+// SetTracingInsecureTransport safely sets the Configuration value for state's 'TracingInsecureTransport' field
+func (st *ConfigState) SetTracingInsecureTransport(v bool) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.TracingInsecureTransport = v
+	st.reloadToViper()
+}
+
+// TracingInsecureTransportFlag returns the flag name for the 'TracingInsecureTransport' field
+func TracingInsecureTransportFlag() string { return "tracing-insecure" }
+
+// GetTracingInsecureTransport safely fetches the value for global configuration 'TracingInsecureTransport' field
+func GetTracingInsecureTransport() bool { return global.GetTracingInsecureTransport() }
+
+// SetTracingInsecureTransport safely sets the value for global configuration 'TracingInsecureTransport' field
+func SetTracingInsecureTransport(v bool) { global.SetTracingInsecureTransport(v) }
+
 // GetSMTPHost safely fetches the Configuration value for state's 'SMTPHost' field
 func (st *ConfigState) GetSMTPHost() (v string) {
 	st.mutex.Lock()
diff --git a/internal/db/bundb/bundb.go b/internal/db/bundb/bundb.go
index 71433dbc2..ecf8752ea 100644
--- a/internal/db/bundb/bundb.go
+++ b/internal/db/bundb/bundb.go
@@ -42,6 +42,7 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/id"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
+	"github.com/superseriousbusiness/gotosocial/internal/tracing"
 	"github.com/uptrace/bun"
 	"github.com/uptrace/bun/dialect/pgdialect"
 	"github.com/uptrace/bun/dialect/sqlitedialect"
@@ -134,6 +135,10 @@ func NewBunDBService(ctx context.Context, state *state.State) (db.DB, error) {
 	// Add database query hook
 	conn.DB.AddQueryHook(queryHook{})
 
+	if config.GetTracingEnabled() {
+		conn.DB.AddQueryHook(tracing.InstrumentBun())
+	}
+
 	// execute sqlite pragmas *after* adding database hook;
 	// this allows the pragma queries to be logged
 	if t == "sqlite" {
diff --git a/internal/middleware/requestid.go b/internal/middleware/requestid.go
index 6e2a83c68..3bf38092f 100644
--- a/internal/middleware/requestid.go
+++ b/internal/middleware/requestid.go
@@ -65,9 +65,8 @@ func generateID() string {
 // AddRequestID returns a gin middleware which adds a unique ID to each request (both response header and context).
 func AddRequestID(header string) gin.HandlerFunc {
 	return func(c *gin.Context) {
-		// Look for existing ID.
 		id := c.GetHeader(header)
-
+		// Have we found anything?
 		if id == "" {
 			// Generate new ID.
 			//
diff --git a/internal/tracing/no_trace.go b/internal/tracing/no_trace.go
new file mode 100644
index 000000000..12f46ce53
--- /dev/null
+++ b/internal/tracing/no_trace.go
@@ -0,0 +1,43 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//go:build notracing
+
+package tracing
+
+import (
+	"errors"
+
+	"github.com/gin-gonic/gin"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/uptrace/bun"
+)
+
+func Initialize() error {
+	if config.GetTracingEnabled() {
+		return errors.New("tracing was disabled at build time")
+	}
+	return nil
+}
+
+func InstrumentGin() gin.HandlerFunc {
+	return func(c *gin.Context) {}
+}
+
+func InstrumentBun() bun.QueryHook {
+	return nil
+}
diff --git a/internal/tracing/tracing.go b/internal/tracing/tracing.go
new file mode 100644
index 000000000..a4b664fb2
--- /dev/null
+++ b/internal/tracing/tracing.go
@@ -0,0 +1,175 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//go:build !notracing
+
+package tracing
+
+import (
+	"context"
+	"fmt"
+
+	"codeberg.org/gruf/go-kv"
+	"github.com/gin-gonic/gin"
+	"github.com/uptrace/bun"
+	"github.com/uptrace/bun/extra/bunotel"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/jaeger"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/resource"
+	"go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
+	"go.opentelemetry.io/otel/semconv/v1.17.0/httpconv"
+	oteltrace "go.opentelemetry.io/otel/trace"
+
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
+	"github.com/superseriousbusiness/gotosocial/internal/log"
+)
+
+const (
+	tracerKey  = "gotosocial-server-tracer"
+	tracerName = "github.com/superseriousbusiness/gotosocial/internal/tracing"
+)
+
+func Initialize() error {
+	if !config.GetTracingEnabled() {
+		return nil
+	}
+
+	insecure := config.GetTracingInsecureTransport()
+
+	var tpo trace.TracerProviderOption
+	switch config.GetTracingTransport() {
+	case "grpc":
+		opts := []otlptracegrpc.Option{
+			otlptracegrpc.WithEndpoint(config.GetTracingEndpoint()),
+		}
+		if insecure {
+			opts = append(opts, otlptracegrpc.WithInsecure())
+		}
+		exp, err := otlptracegrpc.New(context.Background(), opts...)
+		if err != nil {
+			return fmt.Errorf("building tracing exporter: %w", err)
+		}
+		tpo = trace.WithBatcher(exp)
+	case "jaeger":
+		exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(config.GetTracingEndpoint())))
+		if err != nil {
+			return fmt.Errorf("building tracing exporter: %w", err)
+		}
+		tpo = trace.WithBatcher(exp)
+	default:
+		return fmt.Errorf("invalid tracing transport: %s", config.GetTracingTransport())
+	}
+	r, _ := resource.Merge(
+		resource.Default(),
+		resource.NewWithAttributes(
+			semconv.SchemaURL,
+			semconv.ServiceName("GoToSocial"),
+		),
+	)
+
+	tp := trace.NewTracerProvider(
+		tpo,
+		trace.WithResource(r),
+	)
+	otel.SetTracerProvider(tp)
+	propagator := propagation.NewCompositeTextMapPropagator(
+		propagation.TraceContext{},
+		propagation.Baggage{},
+	)
+	otel.SetTextMapPropagator(propagator)
+	log.Hook(func(ctx context.Context, kvs []kv.Field) []kv.Field {
+		span := oteltrace.SpanFromContext(ctx)
+		if span != nil && span.SpanContext().HasTraceID() {
+			return append(kvs, kv.Field{K: "traceID", V: span.SpanContext().TraceID().String()})
+		}
+		return kvs
+	})
+	return nil
+}
+
+// InstrumentGin is a middleware injecting tracing information based on the
+// otelgin implementation found at
+// https://github.com/open-telemetry/opentelemetry-go-contrib/blob/main/instrumentation/github.com/gin-gonic/gin/otelgin/gintrace.go
+func InstrumentGin() gin.HandlerFunc {
+	provider := otel.GetTracerProvider()
+	tracer := provider.Tracer(
+		tracerName,
+		oteltrace.WithInstrumentationVersion(config.GetSoftwareVersion()),
+	)
+	propagator := otel.GetTextMapPropagator()
+	return func(c *gin.Context) {
+		c.Set(tracerKey, tracer)
+		savedCtx := c.Request.Context()
+		defer func() {
+			c.Request = c.Request.WithContext(savedCtx)
+		}()
+		ctx := propagator.Extract(savedCtx, propagation.HeaderCarrier(c.Request.Header))
+		opts := []oteltrace.SpanStartOption{
+			oteltrace.WithAttributes(httpconv.ServerRequest(config.GetHost(), c.Request)...),
+			oteltrace.WithSpanKind(oteltrace.SpanKindServer),
+		}
+		spanName := c.FullPath()
+		if spanName == "" {
+			spanName = fmt.Sprintf("HTTP %s route not found", c.Request.Method)
+		} else {
+			rAttr := semconv.HTTPRoute(spanName)
+			opts = append(opts, oteltrace.WithAttributes(rAttr))
+		}
+		id := gtscontext.RequestID(c.Request.Context())
+		if id != "" {
+			opts = append(opts, oteltrace.WithAttributes(attribute.String("requestID", id)))
+		}
+		ctx, span := tracer.Start(ctx, spanName, opts...)
+		defer span.End()
+
+		// pass the span through the request context
+		c.Request = c.Request.WithContext(ctx)
+
+		// serve the request to the next middleware
+		c.Next()
+
+		status := c.Writer.Status()
+		span.SetStatus(httpconv.ServerStatus(status))
+		if status > 0 {
+			span.SetAttributes(semconv.HTTPStatusCode(status))
+		}
+		if len(c.Errors) > 0 {
+			span.SetAttributes(attribute.String("gin.errors", c.Errors.String()))
+		}
+	}
+}
+
+func InjectRequestID() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		id := gtscontext.RequestID(c.Request.Context())
+		if id != "" {
+			span := oteltrace.SpanFromContext(c.Request.Context())
+			span.SetAttributes(attribute.String("requestID", id))
+		}
+	}
+}
+
+func InstrumentBun() bun.QueryHook {
+	return bunotel.NewQueryHook(
+		bunotel.WithFormattedQueries(true),
+	)
+}