From 27d9c3d0527feecba841a17a400301fa47a5dec0 Mon Sep 17 00:00:00 2001 From: Matthew Walster Date: Thu, 4 Jun 2026 16:48:31 +0000 Subject: [PATCH] otel: add per-query SQL tracing, sampled only on manual sync Flat /api traces showed the request but not the SQL behind it, and there was no way to see ent's queries or the raw sync_status statements as nested spans. Add per-query DB spans via XSAM/otelsql, opened on the shared *sql.DB so ent and the raw queries are both covered. On by default (the data is useful and its volume is bounded); set PDBPLUS_OTEL_SQL=false to disable. otelsql adds no new transitive modules; it depends only on the OpenTelemetry packages already in the graph. Volume is bounded by the existing ParentBased per-route sampler: DB spans inherit their parent request/sync span's decision. The historical concern was the sync path, so scheduled sync cycles are no longer traced: the sampler gains two gates read from the root span's start attributes -- pdbplus.origin=sync drops a span by default, pdbplus.force_sample forces it. The sync worker stamps origin=sync on every cycle and force_sample only when the cycle was started by a manual POST /sync; that flag rides an app-root context value (WithForceTrace) set by the handler. Manual syncs are traced by default; POST /sync?trace=0 opts out. Net: on by default; API-read DB spans follow the per-route rate, scheduled syncs stay trace-free, and a manually-triggered sync is observable end to end including its SQL. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 25 +++++++++++++ cmd/peeringdb-plus/main.go | 13 +++++-- cmd/peeringdb-plus/main_test.go | 2 +- docs/CONFIGURATION.md | 1 + go.mod | 1 + go.sum | 2 ++ internal/config/config.go | 14 ++++++++ internal/database/database.go | 22 ++++++++++-- internal/database/database_test.go | 29 +++++++++++++-- internal/otel/sampler.go | 57 ++++++++++++++++++++++++++++++ internal/otel/sampler_test.go | 40 +++++++++++++++++++++ internal/sync/worker.go | 28 ++++++++++++++- 12 files changed, 225 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e1bb8d2..1a4e855a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,31 @@ Git history (tags `v1.0.0` through `v1.15.0`). ## [Unreleased] +## [1.19.5] — 2026-06-04 + +### Added + +- **Per-query SQL tracing (`PDBPLUS_OTEL_SQL`, on by default).** The shared + `*sql.DB` is opened through XSAM/otelsql so every statement — ent's and the + raw `sync_status` queries — emits a DB span nested under the active + request/sync span, surfacing the SQL behind a request that flat traces + couldn't show. On by default (the data is useful and bounded); set + `PDBPLUS_OTEL_SQL=false` to disable. DB-span volume is bounded by the + existing sampler: API reads inherit `PDBPLUS_OTEL_SAMPLE_RATE`, and + **scheduled sync cycles are no longer traced at all** (so they emit no DB + spans — the historical high-volume concern), while a **manually triggered + `POST /sync` is traced by default** — pass `?trace=0` to opt out. + Implemented via two new sampler gates (`pdbplus.origin=sync` drops scheduled + cycles; `pdbplus.force_sample` force-samples a manual run) and a + `WithForceTrace` context flag threaded from the `/sync` handler. + +### Changed + +- **Scheduled sync cycles are no longer sampled into traces** (previously the + per-route default ~1%). They are dropped by the new `pdbplus.origin=sync` + sampler gate so the sync path stays trace-free unless a sync is triggered + manually via `POST /sync`. Independent of `PDBPLUS_OTEL_SQL`. + ## [1.19.4] — 2026-06-04 ### Changed diff --git a/cmd/peeringdb-plus/main.go b/cmd/peeringdb-plus/main.go index 35155759..596a3576 100644 --- a/cmd/peeringdb-plus/main.go +++ b/cmd/peeringdb-plus/main.go @@ -127,7 +127,7 @@ func main() { } // Open database. - entClient, db, err := database.Open(cfg.DBPath) + entClient, db, err := database.Open(cfg.DBPath, cfg.OTelSQL) if err != nil { logger.Error("failed to open database", slog.Any("error", err)) os.Exit(1) @@ -1202,7 +1202,16 @@ func newSyncHandler(appCtx context.Context, in SyncHandlerInput) http.HandlerFun } // Use application root ctx, NOT r.Context() -- request context // is cancelled when the response is sent, which would kill the sync. - go in.SyncFn(appCtx, mode) + // + // A manually-triggered sync is traced by default (you asked for it, + // you want to see it); ?trace=0 opts out. Scheduled timer syncs are + // never traced (see internal/otel sampler). The force-trace flag rides + // the app root ctx so it reaches the worker's root span. + syncCtx := appCtx + if r.URL.Query().Get("trace") != "0" { + syncCtx = pdbsync.WithForceTrace(appCtx) + } + go in.SyncFn(syncCtx, mode) w.WriteHeader(http.StatusAccepted) fmt.Fprint(w, `{"status":"accepted"}`) } diff --git a/cmd/peeringdb-plus/main_test.go b/cmd/peeringdb-plus/main_test.go index f11ef608..78441e69 100644 --- a/cmd/peeringdb-plus/main_test.go +++ b/cmd/peeringdb-plus/main_test.go @@ -140,7 +140,7 @@ func TestFreshnessFromDB(t *testing.T) { t.Parallel() ctx := context.Background() - _, db, err := database.Open(filepath.Join(t.TempDir(), "freshness.db")) + _, db, err := database.Open(filepath.Join(t.TempDir(), "freshness.db"), false) if err != nil { t.Fatalf("open db: %v", err) } diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 517882ad..0280a959 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -84,6 +84,7 @@ line (`sync interval configured`) — the API key itself is never logged. | Variable | Required | Default | Type | Description | |----------|----------|---------|------|-------------| | `PDBPLUS_OTEL_SAMPLE_RATE` | No | `1.0` | float | Trace sampling ratio for the known app surfaces (`/api/`, `/rest/v1/`, `/peeringdb.v1.`, `/graphql`). Unknown-path / scanner-bait / health-probe ratios are hardcoded — see `docs/ARCHITECTURE.md` § Sampling Matrix. Must be in the inclusive range `[0.0, 1.0]`. Values outside this range are rejected at startup. | +| `PDBPLUS_OTEL_SQL` | No | `true` | bool | Emit a per-query OpenTelemetry DB span (via XSAM/otelsql) for every SQL statement on the shared `*sql.DB` — ent's queries and the raw `sync_status` statements alike. On by default; set `PDBPLUS_OTEL_SQL=false` to disable. DB spans are children of the active request/sync span, so volume is bounded by the existing sampler: API-read spans inherit `PDBPLUS_OTEL_SAMPLE_RATE`; scheduled sync cycles are not traced at all (so emit no DB spans — this was the historical high-volume concern), while a manually-triggered `POST /sync` is traced by default (pass `?trace=0` to opt out). | | `PDBPLUS_LOG_LEVEL` | No | `INFO` | enum | Minimum severity for log records shipped via the OTel logging pipeline (and from there to Loki). Accepted values (case-insensitive, parsed via `slog.Level.UnmarshalText`): `DEBUG`, `INFO`, `WARN`, `ERROR`. The stdout (Fly log) handler is independently gated at INFO and is not affected by this variable. Default `INFO` was chosen so DEBUG records remain local for opt-in debugging without polluting production Loki ingestion volume to keep production Loki ingestion volumes bounded. Invalid values fall back to `INFO` with no error (logging-level config is operator-friendly; fail-fast is normally preferred, but a malformed log level should not take production down). Consumed by `internal/otel/logger.go` `otelLevelFromEnv()` — not parsed by `internal/config`. | | `PDBPLUS_HEAP_WARN_MIB` | No | `400` | integer (MiB) | Peak Go heap (MiB) threshold checked at end of every sync cycle. When `runtime.MemStats.HeapInuse` exceeds this value, the worker emits `slog.Warn("heap threshold crossed", …)` with typed attrs (`peak_heap_bytes`, `heap_warn_bytes`, `heap_over`, etc.). The OTel span attribute `pdbplus.sync.peak_heap_bytes` (Prometheus gauge `pdbplus_sync_peak_heap_bytes`) emits on every cycle regardless. `0` disables the warn. Sustained breach across multiple cycles is the operational signal to revisit the incremental-sync defaults and rollout planning. Default sits comfortably under the 512 MB Fly VM cap so the failure order is `log → app crash → Fly OOM-kill`. Observed baseline (2026-04-17): primary peak ~84 MiB, replicas 58–59 MiB — ~4.5× headroom. **Bare integer only** — no unit suffix (the variable name encodes the unit); `400MB` is rejected. Negative values are rejected at startup. | | `PDBPLUS_RSS_WARN_MIB` | No | `384` | integer (MiB) | Peak OS RSS (MiB) threshold derived from `/proc/self/status` `VmHWM` (Linux only). Same warn semantics as `PDBPLUS_HEAP_WARN_MIB`. The OTel span attr `pdbplus.sync.peak_rss_bytes` (Prometheus gauge `pdbplus_sync_peak_rss_bytes`) is omitted on non-Linux platforms (RSS not available). `0` disables the warn. Bare integer only — no unit suffix. | diff --git a/go.mod b/go.mod index 951ef076..175d34f3 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( entgo.io/ent v0.14.6 github.com/99designs/gqlgen v0.17.90 github.com/KimMachineGun/automemlimit v0.7.5 + github.com/XSAM/otelsql v0.42.0 github.com/a-h/templ v0.3.1020 github.com/charmbracelet/colorprofile v0.4.3 github.com/go-playground/form/v4 v4.3.0 diff --git a/go.sum b/go.sum index 58195a9e..e28af96c 100644 --- a/go.sum +++ b/go.sum @@ -54,6 +54,8 @@ github.com/KimMachineGun/automemlimit v0.7.5 h1:RkbaC0MwhjL1ZuBKunGDjE/ggwAX43Dw github.com/KimMachineGun/automemlimit v0.7.5/go.mod h1:QZxpHaGOQoYvFhv/r4u3U0JTC2ZcOwbSr11UZF46UBM= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/XSAM/otelsql v0.42.0 h1:Li0xF4eJUxG2e0x3D4rvRlys1f27yJKvjTh7ljkUP5o= +github.com/XSAM/otelsql v0.42.0/go.mod h1:4mOrEv+cS1KmKzrvTktvJnstr5GtKSAK+QHvFR9OcpI= github.com/a-h/parse v0.0.0-20250122154542-74294addb73e h1:HjVbSQHy+dnlS6C3XajZ69NYAb5jbGNfHanvm1+iYlo= github.com/a-h/parse v0.0.0-20250122154542-74294addb73e/go.mod h1:3mnrkvGpurZ4ZrTDbYU84xhwXW2TjTKShSwjRi2ihfQ= github.com/a-h/templ v0.3.1020 h1:ypAT/L5ySWEnZ6Zft/5yfoWXYYkhFNvEFOeeqecg4tw= diff --git a/internal/config/config.go b/internal/config/config.go index 82bbd94e..6a8d29f2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -79,6 +79,14 @@ type Config struct { // Configured via PDBPLUS_OTEL_SAMPLE_RATE. Default is 1.0 (always sample). OTelSampleRate float64 + // OTelSQL enables per-query OpenTelemetry DB spans (XSAM/otelsql) on the + // shared *sql.DB. Configured via PDBPLUS_OTEL_SQL. Default true — the data + // is useful and its volume is bounded by the trace sampler; set + // PDBPLUS_OTEL_SQL=false to disable. Scheduled sync cycles are never traced + // regardless (the prior high-volume concern); only a manually-triggered + // POST /sync is — see internal/otel sampler. + OTelSQL bool + // SyncStaleThreshold is the maximum age of sync data before health reports degraded. // Configured via PDBPLUS_SYNC_STALE_THRESHOLD. Default is 24h. SyncStaleThreshold time.Duration @@ -280,6 +288,12 @@ func Load() (*Config, error) { } cfg.OTelSampleRate = sampleRate + otelSQL, err := parseBool("PDBPLUS_OTEL_SQL", true) + if err != nil { + return nil, fmt.Errorf("parsing PDBPLUS_OTEL_SQL: %w", err) + } + cfg.OTelSQL = otelSQL + syncStaleThreshold, err := parseDuration("PDBPLUS_SYNC_STALE_THRESHOLD", 24*time.Hour) if err != nil { return nil, fmt.Errorf("parsing PDBPLUS_SYNC_STALE_THRESHOLD: %w", err) diff --git a/internal/database/database.go b/internal/database/database.go index d3797fbf..52143b4d 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -9,6 +9,8 @@ import ( "entgo.io/ent/dialect" entsql "entgo.io/ent/dialect/sql" + "github.com/XSAM/otelsql" + "go.opentelemetry.io/otel/attribute" "modernc.org/sqlite" "github.com/dotwaffle/peeringdb-plus/ent" @@ -38,13 +40,29 @@ func init() { // - temp_store(MEMORY): keeps sorter and temp tables in RAM. modernc.org/ // sqlite's default is FILE which on Fly hits the rootfs overlay (NOT // tmpfs — verified via /proc/mounts). -func Open(dbPath string) (*ent.Client, *sql.DB, error) { +// +// When traceSQL is true the underlying *sql.DB is opened through XSAM/otelsql, +// so every query — ent's and the raw sync_status statements that share this +// handle — emits an OpenTelemetry span beneath the active request/sync span. +// Controlled by PDBPLUS_OTEL_SQL (default on; set false to disable). Span +// volume is bounded by the trace sampler: scheduled sync cycles are not traced, +// so the high-volume sync path produces no DB spans — see internal/otel sampler. +func Open(dbPath string, traceSQL bool) (*ent.Client, *sql.DB, error) { dsn := fmt.Sprintf( "file:%s?_pragma=foreign_keys(1)&_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)"+ "&_pragma=synchronous(NORMAL)&_pragma=cache_size(-32000)&_pragma=temp_store(MEMORY)", dbPath, ) - db, err := sql.Open("sqlite3", dsn) + var ( + db *sql.DB + err error + ) + if traceSQL { + db, err = otelsql.Open("sqlite3", dsn, + otelsql.WithAttributes(attribute.String("db.system", "sqlite"))) + } else { + db, err = sql.Open("sqlite3", dsn) + } if err != nil { return nil, nil, fmt.Errorf("opening database %s: %w", dbPath, err) } diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 8922ab01..88087e57 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -8,7 +8,7 @@ import ( func TestOpen_Success(t *testing.T) { t.Parallel() dbPath := filepath.Join(t.TempDir(), "test.db") - client, db, err := Open(dbPath) + client, db, err := Open(dbPath, false) if err != nil { t.Fatalf("Open(%q) error: %v", dbPath, err) } @@ -26,7 +26,7 @@ func TestOpen_Success(t *testing.T) { func TestOpen_Pragmas(t *testing.T) { t.Parallel() dbPath := filepath.Join(t.TempDir(), "test.db") - client, db, err := Open(dbPath) + client, db, err := Open(dbPath, false) if err != nil { t.Fatalf("Open(%q) error: %v", dbPath, err) } @@ -55,10 +55,33 @@ func TestOpen_Pragmas(t *testing.T) { } } +// TestOpen_TracedSQL verifies the otelsql-wrapped path (traceSQL=true) opens a +// working handle — the instrumentation is transparent to query execution. +// (Span emission is exercised live with PDBPLUS_OTEL_SQL=1, not here, to avoid +// mutating the global TracerProvider from a parallel test.) +func TestOpen_TracedSQL(t *testing.T) { + t.Parallel() + dbPath := filepath.Join(t.TempDir(), "traced.db") + client, db, err := Open(dbPath, true) + if err != nil { + t.Fatalf("Open(%q, true) error: %v", dbPath, err) + } + defer client.Close() + defer db.Close() + + var n int + if err := db.QueryRow("SELECT 1").Scan(&n); err != nil { + t.Fatalf("query on otelsql-wrapped DB: %v", err) + } + if n != 1 { + t.Errorf("SELECT 1 = %d, want 1", n) + } +} + func TestOpen_PoolConfig(t *testing.T) { t.Parallel() dbPath := filepath.Join(t.TempDir(), "test.db") - client, db, err := Open(dbPath) + client, db, err := Open(dbPath, false) if err != nil { t.Fatalf("Open(%q) error: %v", dbPath, err) } diff --git a/internal/otel/sampler.go b/internal/otel/sampler.go index 43690079..4058844f 100644 --- a/internal/otel/sampler.go +++ b/internal/otel/sampler.go @@ -17,6 +17,19 @@ import ( sdktrace "go.opentelemetry.io/otel/sdk/trace" ) +// Span-start attributes the sync worker stamps on its root span so this +// sampler can gate sync traces independently of HTTP route. AttrSyncOrigin set +// to SyncOriginValue marks a scheduled sync cycle (dropped by default so it +// emits no trace — and, with PDBPLUS_OTEL_SQL on, no per-query DB spans); +// AttrForceSample set true marks a manually-triggered sync (POST /sync) and +// forces the trace to be sampled. Untyped string constants so they compare +// directly against attribute.Key in the ShouldSample scan. +const ( + AttrSyncOrigin = "pdbplus.origin" + SyncOriginValue = "sync" + AttrForceSample = "pdbplus.force_sample" +) + // PerRouteSamplerInput configures NewPerRouteSampler. // // Routes maps URL-path prefix (e.g. "/healthz", "/api/") to a 0–1 @@ -102,6 +115,18 @@ type perRouteSampler struct { // Hot-path allocation is bounded — the entries slice is pre-sorted at // construction time and the SamplingResult is the only allocation. func (s *perRouteSampler) ShouldSample(params sdktrace.SamplingParameters) sdktrace.SamplingResult { + // Sync-trace gating, independent of route: a manual force-sample wins; + // otherwise a sync-origin span (scheduled cycle) is dropped so it emits no + // trace. Non-sync spans fall through to the per-route logic below. + switch syncSampleDecision(params.Attributes) { + case decideForceSample: + return sdktrace.AlwaysSample().ShouldSample(params) + case decideDrop: + return sdktrace.NeverSample().ShouldSample(params) + case decideUnset: + // Not a sync span — fall through to per-route sampling below. + } + path := pathFromAttributes(params.Attributes) if path == "" { return s.defaultSampler.ShouldSample(params) @@ -163,6 +188,38 @@ func pathFromAttributes(attrs []attribute.KeyValue) string { return legacy } +type syncDecision int + +const ( + decideUnset syncDecision = iota + decideForceSample + decideDrop +) + +// syncSampleDecision inspects a span's start attributes for the sync-trace +// gating markers. Returns decideForceSample when AttrForceSample is true +// (manual POST /sync), decideDrop when the span is sync-origin without a force +// flag (scheduled cycle), or decideUnset for everything else (HTTP/internal). +func syncSampleDecision(attrs []attribute.KeyValue) syncDecision { + origin := false + for _, kv := range attrs { + switch kv.Key { + case AttrForceSample: + if kv.Value.AsBool() { + return decideForceSample + } + case AttrSyncOrigin: + if kv.Value.AsString() == SyncOriginValue { + origin = true + } + } + } + if origin { + return decideDrop + } + return decideUnset +} + // Description returns a stable human-readable identifier for OTel debug // output. Includes the marker "PerRouteSampler" and the configured // route count for at-a-glance diagnostics. diff --git a/internal/otel/sampler_test.go b/internal/otel/sampler_test.go index a65a6527..82aec0ff 100644 --- a/internal/otel/sampler_test.go +++ b/internal/otel/sampler_test.go @@ -66,6 +66,46 @@ func TestPerRouteSampler_APIDispatchedToFullRatio(t *testing.T) { } } +// TestPerRouteSampler_SyncTraceGating locks the sync-trace gates: a sync-origin +// span is dropped by default (scheduled cycles emit no trace), force_sample +// always samples (manual POST /sync), and force wins over origin. allOnes +// TraceID makes the assertions deterministic against the route/default ratios. +func TestPerRouteSampler_SyncTraceGating(t *testing.T) { + t.Parallel() + + // DefaultRatio 1.0 would otherwise admit allOnes — proves the gate + // overrides route/default for sync-origin spans. + s := NewPerRouteSampler(PerRouteSamplerInput{ + DefaultRatio: 1.0, + Routes: map[string]float64{"/api/": 1.0}, + }) + + if res := s.ShouldSample(sampleParams(allOnesTraceID, + attribute.String(AttrSyncOrigin, SyncOriginValue))); res.Decision != sdktrace.Drop { + t.Errorf("origin=sync (no force): got %v, want Drop", res.Decision) + } + + // force_sample always samples — here against a 0-ratio default that would + // otherwise drop the all-ones TraceID. + s0 := NewPerRouteSampler(PerRouteSamplerInput{DefaultRatio: 0.0}) + if res := s0.ShouldSample(sampleParams(allOnesTraceID, + attribute.Bool(AttrForceSample, true))); res.Decision != sdktrace.RecordAndSample { + t.Errorf("force_sample: got %v, want RecordAndSample", res.Decision) + } + + if res := s.ShouldSample(sampleParams(allOnesTraceID, + attribute.String(AttrSyncOrigin, SyncOriginValue), + attribute.Bool(AttrForceSample, true))); res.Decision != sdktrace.RecordAndSample { + t.Errorf("force_sample + origin=sync: got %v, want RecordAndSample (force wins)", res.Decision) + } + + // No sync markers: ordinary per-route logic still applies. + if res := s.ShouldSample(sampleParams(allOnesTraceID, + attribute.String("url.path", "/api/net"))); res.Decision != sdktrace.RecordAndSample { + t.Errorf("no markers, /api/ at 1.0: got %v, want RecordAndSample", res.Decision) + } +} + func TestPerRouteSampler_RestV1PrefixMatch(t *testing.T) { t.Parallel() diff --git a/internal/sync/worker.go b/internal/sync/worker.go index bf03dd0b..8e06fdad 100644 --- a/internal/sync/worker.go +++ b/internal/sync/worker.go @@ -580,6 +580,24 @@ func (w *Worker) resolveEffectiveMode(ctx context.Context, configured config.Syn return configured } +// forceTraceKey is the context key carrying the manual-sync force-trace flag. +type forceTraceKey struct{} + +// WithForceTrace marks ctx so the sync cycle run under it force-samples its +// trace, overriding the sampler's default of dropping scheduled-sync traces. +// It is set by the manual POST /sync handler and never by the timer scheduler, +// so an on-demand sync is observable end to end — including its per-query DB +// spans when PDBPLUS_OTEL_SQL is enabled. +func WithForceTrace(ctx context.Context) context.Context { + return context.WithValue(ctx, forceTraceKey{}, true) +} + +// forceTraceFromContext reports whether ctx was marked by WithForceTrace. +func forceTraceFromContext(ctx context.Context) bool { + v, _ := ctx.Value(forceTraceKey{}).(bool) + return v +} + func (w *Worker) Sync(ctx context.Context, mode config.SyncMode) (err error) { ctx = privacy.DecisionContext(ctx, privacy.Allow) // privacy bypass — sole production call site if !w.running.CompareAndSwap(false, true) { @@ -588,7 +606,15 @@ func (w *Worker) Sync(ctx context.Context, mode config.SyncMode) (err error) { } defer w.running.Store(false) - ctx, span := otel.Tracer("sync").Start(ctx, "sync-"+string(mode)) + // Tag the root sync span so the sampler can gate sync traces: origin=sync + // makes scheduled cycles drop by default; a manual POST /sync sets + // force_sample (via WithForceTrace) so that one cycle — and its per-query + // DB spans when PDBPLUS_OTEL_SQL is on — is sampled. + spanAttrs := []attribute.KeyValue{attribute.String(pdbotel.AttrSyncOrigin, pdbotel.SyncOriginValue)} + if forceTraceFromContext(ctx) { + spanAttrs = append(spanAttrs, attribute.Bool(pdbotel.AttrForceSample, true)) + } + ctx, span := otel.Tracer("sync").Start(ctx, "sync-"+string(mode), trace.WithAttributes(spanAttrs...)) defer span.End() start := time.Now()