diff --git a/.changeset/otel-span-customizers.md b/.changeset/otel-span-customizers.md new file mode 100644 index 000000000..a622f55d4 --- /dev/null +++ b/.changeset/otel-span-customizers.md @@ -0,0 +1,12 @@ +--- +"braintrust": minor +"@braintrust/otel": minor +--- + +feat: support OpenTelemetry span customizers + +Run the existing synchronous `onSpanExport` registration on isolated plain OTel snapshots after filtering and origin enrichment. Preserve identity, timing and routing, honor content removal, and fail closed before OTLP serialization by stripping failed spans using the native pipeline's once-only safe diagnostics. Support public processors/exporters, compatibility mode, OTel 1.x/2.x, and ESM/CJS bundles without duplicate customization. + +Keep resource snapshots isolated throughout the customizer chain so a later callback cannot mutate another span's export, including when the callback throws. + +Deduplicate pending-resource waits within each customizer batch while preserving delayed attribute redaction and per-span snapshot isolation. diff --git a/integrations/otel-js/src/index.ts b/integrations/otel-js/src/index.ts index afd66b13b..9555cd32c 100644 --- a/integrations/otel-js/src/index.ts +++ b/integrations/otel-js/src/index.ts @@ -28,17 +28,6 @@ function resetBraintrustCompatCaches() { } export const setupOtelCompat = () => { - // Read braintrust's shared registration directly so this works with any - // supported braintrust version. Keep the message in sync with braintrust. - const spanCustomizers: unknown = (globalThis as Record)[ - Symbol.for("braintrust.spanCustomizers") - ]; - if (Array.isArray(spanCustomizers) && spanCustomizers.length) { - // eslint-disable-next-line no-restricted-properties -- intentional always-visible configuration error. - console.error( - "Braintrust span customizers are not supported with OpenTelemetry compat mode yet.", - ); - } // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/consistent-type-assertions (globalThis as any).BRAINTRUST_CONTEXT_MANAGER = OtelContextManager; // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/consistent-type-assertions diff --git a/integrations/otel-js/src/otel.ts b/integrations/otel-js/src/otel.ts index 6960cdb3d..fa1b4a0fe 100644 --- a/integrations/otel-js/src/otel.ts +++ b/integrations/otel-js/src/otel.ts @@ -1,4 +1,9 @@ import { SpanComponentsV4, SpanObjectTypeV3 } from "braintrust/util"; +import { + customizeOtelSpans, + isCustomizedOtelSpan, + otelCompatFields, +} from "./span-customizer"; import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; import { @@ -354,32 +359,19 @@ class LazyBraintrustOTLPTraceExporter implements SpanExporter { resultCallback: (result: ExportResult) => void, ): void { void this.getExporter() - .then((exporter) => { - const compatibleSpans = spans.map((span) => { - const spanWithCompatFields = span as ReadableSpan & { - instrumentationLibrary?: unknown; - instrumentationScope?: unknown; - parentSpanContext?: { spanId?: string; isRemote?: boolean }; - parentSpanId?: string; - }; - const instrumentationScope = - spanWithCompatFields.instrumentationScope ?? - spanWithCompatFields.instrumentationLibrary; - const instrumentationLibrary = - spanWithCompatFields.instrumentationLibrary ?? - spanWithCompatFields.instrumentationScope; - const parentSpanContext = - spanWithCompatFields.parentSpanContext ?? - (spanWithCompatFields.parentSpanId - ? { - ...span.spanContext(), - spanId: spanWithCompatFields.parentSpanId, - isRemote: false, - } - : undefined); - const parentSpanId = - spanWithCompatFields.parentSpanId ?? - spanWithCompatFields.parentSpanContext?.spanId; + .then(async (exporter) => { + // Customize at the export boundary: upstream processors have already + // filtered, enriched and dropped unsampled spans. + const customizedSpans = await customizeOtelSpans(spans); + const compatibleSpans = customizedSpans.map((span) => { + // Customized snapshots already carry both OTel 1.x and 2.x fields. + if (isCustomizedOtelSpan(span)) return span; + const { + instrumentationScope, + instrumentationLibrary, + parentSpanContext, + parentSpanId, + } = otelCompatFields(span); return new Proxy(span, { get(target, prop) { diff --git a/integrations/otel-js/src/span-customizer.ts b/integrations/otel-js/src/span-customizer.ts new file mode 100644 index 000000000..cd31bae70 --- /dev/null +++ b/integrations/otel-js/src/span-customizer.ts @@ -0,0 +1,407 @@ +import { createTraceState, type SpanContext } from "@opentelemetry/api"; +import type { ReadableSpan } from "@opentelemetry/sdk-trace-base"; + +type RecordData = Record; +const CUSTOMIZED = Symbol.for("braintrust.customizedOtelSpan"); +const RUNTIME_KEY = Symbol.for("braintrust.spanCustomizerRuntime"); +const FAILURE_LOGGED_KEY = Symbol.for("braintrust.spanCustomizerFailureLogged"); + +interface SpanExportAdapter { + snapshot(data: RecordData): RecordData; + restore(data: RecordData): RecordData; + strip(data: RecordData): RecordData; +} + +interface SpanCustomizerRuntime { + isPlainRecord(value: unknown): boolean; + hasCustomizers(): boolean; + customize(data: RecordData, adapter: SpanExportAdapter): RecordData; +} + +// braintrust registers its shared customizer policy under a global symbol. +// Versions without span customizers never register it, so spans pass through. +function runtime(): SpanCustomizerRuntime | undefined { + return (globalThis as Record)[RUNTIME_KEY] as + | SpanCustomizerRuntime + | undefined; +} + +type ResourceCache = Map; + +interface OtelCompatFields { + instrumentationScope?: unknown; + instrumentationLibrary?: unknown; + parentSpanContext?: SpanContext; + parentSpanId?: string; +} + +/** Normalize fields renamed between OTel 1.x and 2.x so either serializer works. */ +export function otelCompatFields(span: ReadableSpan): OtelCompatFields { + const compatible = span as ReadableSpan & OtelCompatFields; + return { + instrumentationScope: + compatible.instrumentationScope ?? compatible.instrumentationLibrary, + instrumentationLibrary: + compatible.instrumentationLibrary ?? compatible.instrumentationScope, + parentSpanContext: + compatible.parentSpanContext ?? + (compatible.parentSpanId + ? { + ...span.spanContext(), + spanId: compatible.parentSpanId, + isRemote: false, + } + : undefined), + parentSpanId: + compatible.parentSpanId ?? compatible.parentSpanContext?.spanId, + }; +} + +/** + * Customize one export batch. Runs at the Braintrust exporter boundary, so + * filtering, enrichment and sampling have already happened upstream. + */ +export async function customizeOtelSpans( + spans: ReadableSpan[], +): Promise { + if (!runtime()?.hasCustomizers()) return spans; + // OTLP serializers wait for async resource detectors. Snapshot only once + // those attributes exist, so hooks can redact them too. A failed detector + // still exports the attributes that resolved, like OTel's batch processor. + // The calling span processor bounds this wait with its export timeout. + const pendingResources = new Set(); + const pendingAttributes: Promise[] = []; + for (const { resource } of spans) { + if ( + resource?.asyncAttributesPending && + resource.waitForAsyncAttributes && + !pendingResources.has(resource) + ) { + pendingResources.add(resource); + pendingAttributes.push(resource.waitForAsyncAttributes().catch(() => {})); + } + } + await Promise.all(pendingAttributes); + // OTLP groups spans by resource object identity, so spans sharing a Resource + // must also share equal adapted resources to keep one ResourceSpans entry. + const resources: ResourceCache = new Map(); + return spans.flatMap((span) => { + try { + return [customizeOtelSpan(span, resources)]; + } catch { + // Malformed spans fail before the strip fallback can run. Fail closed. + reportUnexportableSpan(); + return []; + } + }); +} + +function reportUnexportableSpan(): void { + const shared = globalThis as Record; + if (shared[FAILURE_LOGGED_KEY]) return; + shared[FAILURE_LOGGED_KEY] = true; + // Never log the exception or span: either can contain sensitive data. + // eslint-disable-next-line no-restricted-properties -- intentional always-visible export failure. + console.error( + "[braintrust] Span export customizer failed; dropping an OpenTelemetry span.", + ); +} + +function sameData(a: unknown, b: unknown): boolean { + if (Object.is(a, b)) return true; + if ( + a === null || + b === null || + typeof a !== "object" || + typeof b !== "object" || + Array.isArray(a) !== Array.isArray(b) + ) + return false; + const left = a as RecordData; + const right = b as RecordData; + const keys = Object.keys(left); + return ( + keys.length === Object.keys(right).length && + keys.every( + (key) => + Object.prototype.hasOwnProperty.call(right, key) && + sameData(left[key], right[key]), + ) + ); +} + +// Copy only serialization data: no live SDK objects, getters, functions or cycles +// survive the boundary. This also validates values ignored by OTLP serializers. +// OTel accepts any number as an attribute value, including NaN and Infinity. +function copyData( + value: unknown, + isPlainRecord: (value: unknown) => boolean, + ancestors = new Set(), +): unknown { + if ( + value === undefined || + value === null || + typeof value === "string" || + typeof value === "boolean" || + typeof value === "number" + ) + return value; + if (typeof value !== "object" || ancestors.has(value)) + throw new TypeError("Invalid OTel export data"); + if (!Array.isArray(value) && !isPlainRecord(value)) + throw new TypeError("Invalid OTel export container"); + ancestors.add(value); + try { + if (Array.isArray(value)) + return value.map((item) => copyData(item, isPlainRecord, ancestors)); + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [ + key, + copyData(item, isPlainRecord, ancestors), + ]), + ); + } finally { + ancestors.delete(value); + } +} + +function record(value: unknown): RecordData { + if (value === null || typeof value !== "object" || Array.isArray(value)) + throw new TypeError("Invalid OTel export object"); + return value as RecordData; +} + +function attributes(value: unknown): RecordData { + const result = record(value ?? {}); + for (const item of Object.values(result)) { + const values = Array.isArray(item) ? item : [item]; + if ( + !values.every( + (v) => + v === null || + v === undefined || + typeof v === "string" || + typeof v === "boolean" || + typeof v === "number", + ) + ) + throw new TypeError("Invalid OTel attribute"); + } + return result; +} + +function time(value: unknown): void { + if ( + !Array.isArray(value) || + value.length !== 2 || + !value.every((v) => typeof v === "number" && Number.isFinite(v)) + ) + throw new TypeError("Invalid OTel time"); +} + +function droppedCount(value: unknown): void { + if ( + value !== undefined && + (!Number.isInteger(value) || (value as number) < 0) + ) + throw new TypeError("Invalid OTel dropped count"); +} + +function validateContext(value: unknown): RecordData { + const result = record(value); + if ( + typeof result.traceId !== "string" || + !/^[0-9a-f]{32}$/i.test(result.traceId) || + typeof result.spanId !== "string" || + !/^[0-9a-f]{16}$/i.test(result.spanId) || + !Number.isInteger(result.traceFlags) || + (result.traceState !== undefined && + typeof result.traceState !== "string") || + (result.isRemote !== undefined && typeof result.isRemote !== "boolean") + ) + throw new TypeError("Invalid OTel span context"); + return result; +} + +function contextData(context: SpanContext | undefined): RecordData | undefined { + if (!context) return undefined; + return { + traceId: context.traceId, + spanId: context.spanId, + traceFlags: context.traceFlags, + ...(context.isRemote !== undefined ? { isRemote: context.isRemote } : {}), + ...(context.traceState + ? { traceState: context.traceState.serialize() } + : {}), + }; +} + +/** Customize an isolated, plain OTel snapshot, never a live ReadableSpan. */ +function customizeOtelSpan( + span: ReadableSpan, + resources: ResourceCache, +): ReadableSpan { + const policy = runtime(); + if (isCustomizedOtelSpan(span) || !policy) return span; + const copy = (value: unknown) => copyData(value, policy.isPlainRecord); + const compatible = otelCompatFields(span); + // Protocol snapshots are kept private, not shared with any hook. + const identity = { + spanContext: contextData(span.spanContext()), + parentSpanId: compatible.parentSpanId, + parentSpanContext: contextData(compatible.parentSpanContext), + startTime: [...span.startTime], + endTime: [...span.endTime], + duration: [...span.duration], + }; + const parent = span.attributes["braintrust.parent"]; + const routing = parent === undefined ? {} : { "braintrust.parent": parent }; + // Validate after each callback so an invalid result stops the chain. + const restore = (data: RecordData): RecordData => { + const result = record(copy(data)); + result.spanContext = copy(identity.spanContext); + result.parentSpanId = identity.parentSpanId; + result.parentSpanContext = copy(identity.parentSpanContext); + result.startTime = [...identity.startTime]; + result.endTime = [...identity.endTime]; + result.duration = [...identity.duration]; + result.attributes = { ...attributes(result.attributes), ...routing }; + if (parent === undefined) + delete (result.attributes as RecordData)["braintrust.parent"]; + result.name ??= ""; + result.kind ??= 0; + result.status ??= { code: 0 }; + result.events ??= []; + result.links ??= []; + result.resource ??= { attributes: {} }; + result.instrumentationScope ??= { name: "" }; + if ( + typeof result.name !== "string" || + !Number.isInteger(result.kind) || + (result.kind as number) < 0 || + (result.kind as number) > 4 + ) + throw new TypeError("Invalid OTel span"); + const status = record(result.status); + if ( + ![0, 1, 2].includes(status.code as number) || + (status.message !== undefined && typeof status.message !== "string") + ) + throw new TypeError("Invalid OTel status"); + const resource = record(result.resource); + resource.attributes = attributes(resource.attributes); + const scope = record(result.instrumentationScope); + scope.name ??= ""; + if ( + typeof scope.name !== "string" || + (scope.version !== undefined && typeof scope.version !== "string") || + (scope.schemaUrl !== undefined && typeof scope.schemaUrl !== "string") || + (resource.schemaUrl !== undefined && + typeof resource.schemaUrl !== "string") + ) + throw new TypeError("Invalid OTel metadata"); + if (scope.attributes !== undefined) + scope.attributes = attributes(scope.attributes); + if (!Array.isArray(result.events) || !Array.isArray(result.links)) + throw new TypeError("Invalid OTel events or links"); + for (const item of result.events) { + const event = record(item); + if (typeof event.name !== "string") + throw new TypeError("Invalid OTel event"); + time(event.time); + event.attributes = attributes(event.attributes); + droppedCount(event.droppedAttributesCount); + } + for (const item of result.links) { + const link = record(item); + validateContext(link.context); + link.attributes = attributes(link.attributes); + droppedCount(link.droppedAttributesCount); + } + validateContext(result.spanContext); + for (const key of [ + "droppedAttributesCount", + "droppedEventsCount", + "droppedLinksCount", + ]) { + result[key] ??= 0; + droppedCount(result[key]); + } + return result; + }; + const data = policy.customize( + {}, + { + snapshot() { + return record( + copy({ + ...identity, + name: span.name, + kind: span.kind, + attributes: span.attributes, + events: span.events, + links: span.links.map((link) => ({ + ...link, + context: contextData(link.context), + })), + status: span.status, + resource: { + attributes: span.resource.attributes, + schemaUrl: (span.resource as unknown as RecordData).schemaUrl, + }, + instrumentationScope: compatible.instrumentationScope ?? { + name: "", + }, + droppedAttributesCount: span.droppedAttributesCount, + droppedEventsCount: span.droppedEventsCount, + droppedLinksCount: span.droppedLinksCount, + }), + ); + }, + restore, + strip: () => + restore({ + attributes: { + "braintrust.context_json": '{"customizer_error":true}', + }, + }), + }, + ); + if (Object.keys(data).length === 0) return span; + // Share resources only after all hooks finish so later hooks cannot mutate + // resource objects already attached to another span's export snapshot. + const adapted = resources.get(span.resource) ?? []; + resources.set(span.resource, adapted); + const shared = adapted.find((resource) => sameData(resource, data.resource)); + if (shared) data.resource = shared; + else adapted.push(record(data.resource)); + // Adapt only the customized snapshot. Never proxy/fall back to the original: + // removal of resource, scope, events, etc. must really remove that content. + const toContext = (value: unknown): SpanContext => { + const context = record(value); + return { + ...context, + ...(typeof context.traceState === "string" + ? { traceState: createTraceState(context.traceState) } + : {}), + } as unknown as SpanContext; + }; + const exported = { + ...data, + spanContext: () => toContext(data.spanContext), + parentSpanContext: + data.parentSpanContext && toContext(data.parentSpanContext), + instrumentationLibrary: data.instrumentationScope, + links: (data.links as RecordData[]).map((link) => ({ + ...link, + context: toContext(link.context), + })), + ended: true, + }; + Object.defineProperty(exported, CUSTOMIZED, { value: true }); + return exported as unknown as ReadableSpan; +} + +export function isCustomizedOtelSpan(span: ReadableSpan): boolean { + return !!(span as unknown as Record)[CUSTOMIZED]; +} diff --git a/integrations/otel-js/src/span-customizers.test.ts b/integrations/otel-js/src/span-customizers.test.ts index 86655e324..7585b8f5a 100644 --- a/integrations/otel-js/src/span-customizers.test.ts +++ b/integrations/otel-js/src/span-customizers.test.ts @@ -1,73 +1,939 @@ -import { afterEach, describe, expect, test, vi } from "vitest"; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + test, + vi, +} from "vitest"; +import { createServer, type Server } from "node:http"; +import { createRequire } from "node:module"; +import type { AddressInfo } from "node:net"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import vm from "node:vm"; +import * as braintrust from "braintrust"; +import { context, createTraceState, trace } from "@opentelemetry/api"; import { BasicTracerProvider, InMemorySpanExporter, + SamplingDecision, SimpleSpanProcessor, + type ReadableSpan, } from "@opentelemetry/sdk-trace-base"; -import { setupOtelCompat, resetOtelCompat, BraintrustSpanProcessor } from "."; +import { + setupOtelCompat, + resetOtelCompat, + BraintrustSpanProcessor, + BraintrustExporter, +} from "."; import { createTracerProvider } from "../tests/utils"; -// Braintrust stores configured span customizers under this shared symbol. -const SPAN_CUSTOMIZERS_KEY = Symbol.for("braintrust.spanCustomizers"); -const shared = globalThis as Record; +type Data = Record; +const shared = globalThis as Record; +const customizersKey = Symbol.for("braintrust.spanCustomizers"); +const failureKey = Symbol.for("braintrust.spanCustomizerFailureLogged"); +const require = createRequire(`${process.cwd()}/package.json`); -describe("setupOtelCompat with span customizers", () => { +describe("OpenTelemetry span customizers at the HTTP boundary", () => { + let server: Server; + let apiUrl: string; + let bodies: Data[]; + let headers: Data[]; + let retryRemaining = 0; + beforeAll(async () => { + server = createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => { + bodies.push(JSON.parse(Buffer.concat(chunks).toString())); + headers.push(req.headers); + if (retryRemaining-- > 0) { + res.writeHead(503, { "retry-after": "0" }).end(); + return; + } + res.writeHead(200, { "content-type": "application/json" }).end("{}"); + }); + }); + await new Promise((resolve) => + server.listen(0, "127.0.0.1", resolve), + ); + apiUrl = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + }); + afterAll(async () => { + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + }); + beforeEach(() => { + bodies = []; + headers = []; + retryRemaining = 0; + delete shared[failureKey]; + }); afterEach(() => { - delete shared[SPAN_CUSTOMIZERS_KEY]; + delete shared[customizersKey]; + delete shared[failureKey]; resetOtelCompat(); + trace.disable(); vi.restoreAllMocks(); }); - test("logs once per setup attempt and continues exporting spans", async () => { - const errorLog = vi.spyOn(console, "error").mockImplementation(() => {}); - const onSpanExport = vi.fn((data: unknown) => data); - const customizers = [{ onSpanExport }]; - shared[SPAN_CUSTOMIZERS_KEY] = customizers; + function received(): Data[] { + return bodies.flatMap((body) => + body.resourceSpans.flatMap((resource: Data) => + resource.scopeSpans.flatMap((scope: Data) => scope.spans), + ), + ); + } - setupOtelCompat(); - expect(errorLog).toHaveBeenCalledTimes(1); - expect(shared.BRAINTRUST_CONTEXT_MANAGER).toBeDefined(); - expect(shared.BRAINTRUST_ID_GENERATOR).toBeDefined(); - expect(shared.BRAINTRUST_SPAN_COMPONENT).toBeDefined(); - expect(shared[SPAN_CUSTOMIZERS_KEY]).toBe(customizers); + async function fixture(): Promise { + const memory = new InMemorySpanExporter(); + const provider = createTracerProvider(BasicTracerProvider, [ + new SimpleSpanProcessor(memory), + ]); + const tracer = provider.getTracer("PRIVATE-scope", "PRIVATE-version"); + const root = tracer.startSpan("PRIVATE-root", { + attributes: { + "braintrust.parent": "project_id:routed", + "braintrust.input_json": '"PRIVATE-input"', + "braintrust.context_json": '{"secret":"PRIVATE-context"}', + values: ["PRIVATE-array"], + }, + }); + const child = tracer.startSpan( + "PRIVATE-child", + { + links: [ + { + context: root.spanContext(), + attributes: { secret: "PRIVATE-link" }, + }, + ], + }, + trace.setSpan(context.active(), root), + ); + root.addEvent("PRIVATE-event", { secret: "PRIVATE-event-attribute" }); + root.setStatus({ code: 2, message: "PRIVATE-status" }); + child.end(); + root.end(); + await provider.forceFlush(); + const spans = [...memory.getFinishedSpans()]; + await provider.shutdown(); + return spans; + } + + async function send(spans: ReadableSpan[]): Promise { + const exporter = new BraintrustExporter({ + apiKey: "test", + apiUrl, + parent: "project_name:default", + }); + try { + await new Promise((resolve, reject) => + exporter.export(spans, (result) => + result.code ? reject(result.error) : resolve(), + ), + ); + } finally { + await exporter.shutdown(); + } + } + + test("replacement snapshots remove content without changing live spans, identity, timing or routing", async () => { + const spans = await fixture(); + const seen: Data[] = []; + shared[customizersKey] = [ + { + onSpanExport(data: Data) { + expect(Object.getPrototypeOf(data)).toBe(Object.prototype); + expect(typeof data.spanContext).toBe("object"); + expect( + JSON.parse(data.attributes["braintrust.context_json"]), + ).toHaveProperty("span_origin"); + seen.push(structuredClone(data)); + data.spanContext.spanId = "aaaaaaaaaaaaaaaa"; + data.startTime[0] = 0; + if (data.attributes.values) data.attributes.values[0] = "mutated"; + data.resource.attributes.secret = "mutated"; + data.instrumentationScope.name = "mutated"; + return { + name: "redacted", + attributes: { + clean: true, + "braintrust.parent": "project_id:wrong", + }, + }; + }, + }, + { + onSpanExport(data: Data) { + expect(data.spanContext.spanId).not.toBe("aaaaaaaaaaaaaaaa"); + expect(data.events).toEqual([]); + return data; + }, + }, + ]; + await send(spans); + expect(seen).toHaveLength(2); + const output = received(); + expect(output).toHaveLength(2); + for (let i = 0; i < spans.length; i++) { + const original = spans[i]; + const exported = output.find( + (entry) => entry.spanId === original.spanContext().spanId, + )!; + expect(exported.traceId).toBe(original.spanContext().traceId); + expect(exported.name).toBe("redacted"); + expect( + BigInt(exported.endTimeUnixNano) - BigInt(exported.startTimeUnixNano), + ).toBe( + BigInt(original.duration[0]) * 1_000_000_000n + + BigInt(original.duration[1]), + ); + expect(exported.events ?? []).toEqual([]); + expect(exported.links ?? []).toEqual([]); + expect(exported.status?.message).toBeUndefined(); + expect(original.name).toContain("PRIVATE"); + expect(original.resource.attributes.secret).toBeUndefined(); + } + const root = output.find((entry) => + entry.attributes.some((attr: Data) => attr.key === "braintrust.parent"), + )!; + expect(root.attributes).toContainEqual({ + key: "braintrust.parent", + value: { stringValue: "project_id:routed" }, + }); + expect(output.find((entry) => entry !== root)?.parentSpanId).toBe( + root.spanId, + ); + expect(JSON.stringify(bodies)).not.toContain("PRIVATE"); + expect(JSON.stringify(bodies)).not.toContain("mutated"); + expect(headers[0]["x-bt-parent"]).toBe("project_name:default"); + expect( + spans.find((span) => span.name === "PRIVATE-root")?.attributes.values, + ).toEqual(["PRIVATE-array"]); + }); + + test.each([undefined, [], [{}]])( + "keeps the no-hook path unchanged (%j)", + async (customizers) => { + shared[customizersKey] = customizers; + await send(await fixture()); + expect(received().map((span) => span.name)).toEqual([ + "PRIVATE-child", + "PRIVATE-root", + ]); + expect(JSON.stringify(bodies)).toContain("PRIVATE-event"); + }, + ); + + test("redacts shared async resource attributes before export", async () => { + const spans = await fixture(); + const resource = { + attributes: {} as Record, + asyncAttributesPending: true, + async waitForAsyncAttributes() { + await Promise.resolve(); + resource.attributes = { secret: "PRIVATE-resource", clean: "kept" }; + resource.asyncAttributesPending = false; + }, + }; + const hook = vi.fn((data: Data) => { + expect(data.resource.attributes.secret).toBe("PRIVATE-resource"); + delete data.resource.attributes.secret; + return { name: "redacted", resource: data.resource }; + }); + shared[customizersKey] = [{ onSpanExport: hook }]; + const pendingSpans = spans.map( + (span) => + new Proxy(span, { + get(target, key) { + if (key === "resource") return resource; + const value = Reflect.get(target, key, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }), + ); + await send(pendingSpans); + expect(hook).toHaveBeenCalledTimes(2); + expect(received().map((span) => span.name)).toEqual([ + "redacted", + "redacted", + ]); + expect(bodies[0].resourceSpans).toHaveLength(1); + expect(bodies[0].resourceSpans[0].resource.attributes).toEqual([ + { key: "clean", value: { stringValue: "kept" } }, + ]); + expect(resource.attributes.secret).toBe("PRIVATE-resource"); + expect(JSON.stringify(bodies)).not.toContain("PRIVATE"); + }); - const exporter = new InMemorySpanExporter(); + test("accepts cross-realm and null-prototype replacements and serializes added events and links", async () => { + const spans = await fixture(); + shared[customizersKey] = [ + { + onSpanExport: () => + vm.runInNewContext( + '({ name: "cross-realm", attributes: { clean: ["safe"] } })', + ), + }, + { + onSpanExport(data: Data) { + const result = Object.assign(Object.create(null), data); + result.events = [ + { name: "safe-event", time: [1, 2], attributes: { clean: true } }, + ]; + result.links = [ + { context: data.spanContext, attributes: { clean: true } }, + ]; + return result; + }, + }, + ]; + await send(spans); + expect(received()).toHaveLength(2); + expect(received()[0].events[0].name).toBe("safe-event"); + expect(received()[0].links[0].spanId).toBe(spans[0].spanContext().spanId); + expect(JSON.stringify(bodies)).not.toContain("PRIVATE"); + }); + + test.each([ + [ + "throw", + () => { + throw new Error("PRIVATE-error"); + }, + ], + ["undefined", () => undefined], + ["null", () => null], + ["promise", () => Promise.reject(new Error("PRIVATE-promise"))], + ["array", () => []], + ["class", () => new (class Result {})()], + [ + "cyclic", + (data: Data) => { + data.attributes.cycle = data; + return data; + }, + ], + [ + "nested attribute", + (data: Data) => { + data.attributes.bad = { private: true }; + return data; + }, + ], + [ + "event", + (data: Data) => { + data.events = [{ name: "PRIVATE", time: "bad" }]; + return data; + }, + ], + [ + "link", + (data: Data) => { + data.links = [{ context: {} }]; + return data; + }, + ], + [ + "status", + (data: Data) => { + data.status = { code: "bad", message: "PRIVATE" }; + return data; + }, + ], + [ + "resource", + (data: Data) => { + data.resource = { attributes: { bad: {} } }; + return data; + }, + ], + [ + "scope", + (data: Data) => { + data.instrumentationScope = { name: {} }; + return data; + }, + ], + ])( + "fails closed for %s while exporting other records", + async (_name, fail) => { + const spans = await fixture(); + const errorLog = vi.spyOn(console, "error").mockImplementation(() => {}); + const subsequent = vi.fn((data: Data) => data); + shared[customizersKey] = [ + { + onSpanExport(data: Data) { + if (data.name === "PRIVATE-root") return fail(data); + return { name: "healthy" }; + }, + }, + { onSpanExport: subsequent }, + ]; + await send(spans); + await send(spans); + expect(subsequent).toHaveBeenCalledTimes(2); + expect(errorLog).toHaveBeenCalledTimes(1); + expect(JSON.stringify(errorLog.mock.calls)).not.toContain("PRIVATE"); + const output = received(); + expect(output).toHaveLength(4); + const stripped = output.filter((entry) => entry.name === ""); + expect(stripped).toHaveLength(2); + const original = spans.find((span) => span.name === "PRIVATE-root")!; + for (const span of stripped) { + expect(span.spanId).toBe(original.spanContext().spanId); + expect(span.traceId).toBe(original.spanContext().traceId); + expect( + BigInt(span.endTimeUnixNano) - BigInt(span.startTimeUnixNano), + ).toBe( + BigInt(original.duration[0]) * 1_000_000_000n + + BigInt(original.duration[1]), + ); + expect(span.attributes).toEqual( + expect.arrayContaining([ + { + key: "braintrust.context_json", + value: { stringValue: '{"customizer_error":true}' }, + }, + { + key: "braintrust.parent", + value: { stringValue: "project_id:routed" }, + }, + ]), + ); + expect(span.attributes).toHaveLength(2); + expect(span.events ?? []).toEqual([]); + expect(span.links ?? []).toEqual([]); + expect(span.status?.message).toBeUndefined(); + } + expect(output.filter((entry) => entry.name === "healthy")).toHaveLength( + 2, + ); + for (const body of bodies) { + for (const resource of body.resourceSpans) { + expect(resource.resource.attributes ?? []).toEqual([]); + for (const scope of resource.scopeSpans) { + expect(scope.scope.name ?? "").toBe(""); + expect(scope.scope.version).toBeUndefined(); + } + } + } + expect(JSON.stringify(bodies)).not.toContain("PRIVATE"); + }, + ); + + test("filters before customization, enriches origin first, and never changes another consumer", async () => { + const other = new InMemorySpanExporter(); + const hook = vi.fn((data: Data) => { + expect(data.attributes["braintrust.context_json"]).toContain("sdk"); + return { name: "redacted" }; + }); + shared[customizersKey] = [{ onSpanExport: hook }]; + const processor = new BraintrustSpanProcessor({ + apiKey: "test", + apiUrl, + parent: "project_name:test", + filterAISpans: true, + }); const provider = createTracerProvider(BasicTracerProvider, [ - new BraintrustSpanProcessor({ - parent: "project_name:customizer-compat", - _spanProcessor: new SimpleSpanProcessor(exporter), - }), + processor, + new SimpleSpanProcessor(other), ]); try { - const tracer = provider.getTracer("customizer-compat"); - tracer.startSpan("first").end(); - tracer.startSpan("second").end(); + const tracer = provider.getTracer("consumer"); + tracer.startSpan("dropped").end(); + tracer.startSpan("gen_ai.PRIVATE").end(); await provider.forceFlush(); - expect(exporter.getFinishedSpans().map((span) => span.name)).toEqual([ - "first", - "second", + expect(hook).toHaveBeenCalledTimes(1); + expect(received().map((span) => span.name)).toEqual(["redacted"]); + expect(other.getFinishedSpans().map((span) => span.name)).toEqual([ + "dropped", + "gen_ai.PRIVATE", ]); - expect(onSpanExport).not.toHaveBeenCalled(); - expect(errorLog).toHaveBeenCalledTimes(1); + } finally { + await provider.shutdown(); + } + }); - setupOtelCompat(); - expect(errorLog).toHaveBeenCalledTimes(2); - expect(shared[SPAN_CUSTOMIZERS_KEY]).toBe(customizers); + test("composed processor/exporter invokes once and does not re-enrich stripped records", async () => { + const hook = vi.fn(() => { + throw new Error("PRIVATE"); + }); + vi.spyOn(console, "error").mockImplementation(() => {}); + shared[customizersKey] = [{ onSpanExport: hook }]; + const exporter = new BraintrustExporter({ + apiKey: "test", + apiUrl, + parent: "project_name:test", + }); + const processor = new BraintrustSpanProcessor({ + _spanProcessor: new SimpleSpanProcessor(exporter), + }); + const spans = await fixture(); + processor.onEnd(spans[0]); + await processor.forceFlush(); + await processor.shutdown(); + expect(hook).toHaveBeenCalledTimes(1); + expect(received()).toHaveLength(1); + expect(received()[0].attributes).toEqual([ + { + key: "braintrust.context_json", + value: { stringValue: '{"customizer_error":true}' }, + }, + ]); + expect(JSON.stringify(bodies)).not.toContain("PRIVATE"); + }); + + test("HTTP retries reuse the stripped snapshot without rerunning hooks", async () => { + retryRemaining = 1; + const hook = vi.fn(() => { + throw new Error("PRIVATE"); + }); + const errorLog = vi.spyOn(console, "error").mockImplementation(() => {}); + shared[customizersKey] = [{ onSpanExport: hook }]; + const [span] = await fixture(); + await send([span]); + expect(hook).toHaveBeenCalledTimes(1); + expect(errorLog).toHaveBeenCalledTimes(1); + expect(bodies).toHaveLength(2); + expect(bodies[1]).toEqual(bodies[0]); + expect(JSON.stringify(bodies)).not.toContain("PRIVATE"); + expect(received()[0].attributes).toEqual([ + { + key: "braintrust.context_json", + value: { stringValue: '{"customizer_error":true}' }, + }, + ]); + }); + + test("compatibility mode customizes native logger/experiment and OTel roots and children", async () => { + const processor = new BraintrustSpanProcessor({ + apiKey: "test", + apiUrl, + parent: "project_name:test", + }); + const provider = createTracerProvider(BasicTracerProvider, [processor]); + trace.setGlobalTracerProvider(provider); + const hook = vi.fn((data: Data) => + "attributes" in data + ? { name: "redacted", attributes: { clean: true } } + : { output: "redacted" }, + ); + shared[customizersKey] = [{ onSpanExport: hook }]; + await braintrust._exportsForTestingOnly.simulateLoginForTests(); + const memory = braintrust._exportsForTestingOnly.useTestBackgroundLogger(); + setupOtelCompat(); + try { + const logger = braintrust.initLogger({ + projectId: "test", + projectName: "test", + }); + const experiment = braintrust._exportsForTestingOnly.initTestExperiment( + "experiment", + "test", + ); + for (const parent of [logger, experiment]) { + const root = parent.startSpan({ name: "PRIVATE" }); + const child = root.startSpan({ name: "PRIVATE-child" }); + child.log({ input: "PRIVATE-input" }); + child.end(); + root.end(); + } + const records = await memory.drain(); + expect(records).toHaveLength(4); + expect(JSON.stringify(records)).not.toContain("PRIVATE"); + const tracer = provider.getTracer("compat"); + const root = tracer.startSpan("PRIVATE"); + tracer + .startSpan("PRIVATE-child", {}, trace.setSpan(context.active(), root)) + .end(); + root.end(); + await provider.forceFlush(); + expect( + hook.mock.calls.filter(([data]) => "attributes" in data), + ).toHaveLength(2); + // Native spans may produce several incremental records before merging. + expect( + hook.mock.calls.filter(([data]) => "id" in data).length, + ).toBeGreaterThanOrEqual(4); + expect(received()).toHaveLength(2); + expect(JSON.stringify(bodies)).not.toContain("PRIVATE"); } finally { await provider.shutdown(); + braintrust._exportsForTestingOnly.clearTestBackgroundLogger(); + braintrust._exportsForTestingOnly.simulateLogoutForTests(); } }); - test.each([undefined, []])( - "allows setup when customizers are cleared (%j)", - (customizers) => { + test.each([true, false])( + "shares once-only diagnostics across CJS native and ESM OTel exports (native first: %s)", + async (nativeFirst) => { + const bt = require("braintrust") as typeof braintrust; const errorLog = vi.spyOn(console, "error").mockImplementation(() => {}); - shared[SPAN_CUSTOMIZERS_KEY] = customizers; + shared[customizersKey] = [ + { + onSpanExport() { + throw new Error("PRIVATE"); + }, + }, + ]; + await bt._exportsForTestingOnly.simulateLoginForTests(); + const memory = bt._exportsForTestingOnly.useTestBackgroundLogger(); + const native = async () => { + const span = bt + .initLogger({ projectName: "native", projectId: "native" }) + .startSpan({ name: "PRIVATE" }); + span.log({ output: "PRIVATE" }); + span.end(); + const records = await memory.drain(); + expect(records[0]).toHaveProperty("context", { + customizer_error: true, + }); + expect(JSON.stringify(records)).not.toContain("PRIVATE"); + }; + try { + const spans = await fixture(); + if (nativeFirst) await native(); + await send(spans); + if (!nativeFirst) await native(); + expect(errorLog).toHaveBeenCalledTimes(1); + expect(received()).toHaveLength(2); + expect(JSON.stringify(bodies)).not.toContain("PRIVATE"); + } finally { + bt._exportsForTestingOnly.clearTestBackgroundLogger(); + bt._exportsForTestingOnly.simulateLogoutForTests(); + } + }, + ); + test("standalone ESM registration configures the CJS OTel bundle before initialization", async () => { + // A fresh process is necessary: this worker has initialized instrumentation. + // The bootstrap's delayed imports intentionally test module loading order. + const script = ` + import { configureInstrumentation } from "braintrust/instrumentation"; + import { createRequire } from "node:module"; + import { promisify } from "node:util"; + const require = createRequire(import.meta.url); + let calls = 0; + configureInstrumentation({ spanCustomizers: [{ onSpanExport(data) { + calls++; + return { name: "bootstrap-redacted" }; + } }] }); + const { BraintrustExporter, setupOtelCompat } = require("@braintrust/otel"); setupOtelCompat(); - expect(shared.BRAINTRUST_CONTEXT_MANAGER).toBeDefined(); - expect(shared.BRAINTRUST_ID_GENERATOR).toBeDefined(); - expect(shared.BRAINTRUST_SPAN_COMPONENT).toBeDefined(); - expect(errorLog).not.toHaveBeenCalled(); + const exporter = new BraintrustExporter({ + apiKey: "test", apiUrl: ${JSON.stringify(apiUrl)}, parent: "project_name:bootstrap" + }); + const span = { + name: "PRIVATE", spanContext: () => ({ traceId: "1".repeat(32), spanId: "2".repeat(16), traceFlags: 1 }), + startTime: [1, 0], endTime: [2, 0], duration: [1, 0], + attributes: {}, events: [], links: [], status: { code: 0 }, kind: 0, + resource: { attributes: {} }, instrumentationScope: { name: "PRIVATE" }, + droppedAttributesCount: 0, droppedEventsCount: 0, droppedLinksCount: 0 + }; + await promisify((done) => { + exporter.export([span], result => done( + result.code ? result.error ?? new Error("Export failed") : null + )); + })(); + await exporter.shutdown(); + if (calls !== 1) throw new Error("Expected one callback"); + `; + await promisify(execFile)( + process.execPath, + ["--input-type=module", "-e", script], + { cwd: process.cwd() }, + ); + expect(received().map((span) => span.name)).toEqual(["bootstrap-redacted"]); + expect(JSON.stringify(bodies)).not.toContain("PRIVATE"); + }); + + test("a never-resolving async resource cannot hang flush", async () => { + const [span] = await fixture(); + const hook = vi.fn((data: Data) => data); + shared[customizersKey] = [{ onSpanExport: hook }]; + const pendingSpan = new Proxy(span, { + get(target, key) { + if (key === "resource") + return { + attributes: {}, + asyncAttributesPending: true, + waitForAsyncAttributes: () => new Promise(() => {}), + }; + const value = Reflect.get(target, key, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); + const processor = new BraintrustSpanProcessor({ + apiKey: "test", + apiUrl, + parent: "project_name:test", + }); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + try { + processor.onEnd(pendingSpan); + const flushed = processor.forceFlush().then( + () => "settled", + () => "settled", + ); + // BatchSpanProcessor's default export timeout bounds the wait. + await vi.advanceTimersByTimeAsync(30_000); + expect(await flushed).toBe("settled"); + } finally { + vi.useRealTimers(); + } + expect(hook).not.toHaveBeenCalled(); + }); + + test("composed inner processors filter the uncustomized span", async () => { + const hook = vi.fn((data: Data) => { + delete data.attributes["gen_ai.PRIVATE"]; + return data; + }); + shared[customizersKey] = [{ onSpanExport: hook }]; + const exporter = new BraintrustExporter({ + apiKey: "test", + apiUrl, + parent: "project_name:test", + filterAISpans: true, + }); + const processor = new BraintrustSpanProcessor({ + _spanProcessor: new SimpleSpanProcessor(exporter), + }); + const provider = createTracerProvider(BasicTracerProvider, [processor]); + try { + provider + .getTracer("composed") + .startSpan("ai-span", { attributes: { "gen_ai.PRIVATE": "value" } }) + .end(); + await provider.forceFlush(); + } finally { + await provider.shutdown(); + } + expect(hook).toHaveBeenCalledTimes(1); + expect(received().map((span) => span.name)).toEqual(["ai-span"]); + expect(JSON.stringify(bodies)).not.toContain("PRIVATE"); + }); + + test.each([ + ["changed", false], + ["stripped", true], + ])( + "trace state and remote flag are protocol identity (%s)", + async (_name, fail) => { + const [span] = await fixture(); + const withTraceState = new Proxy(span, { + get(target, key) { + if (key === "spanContext") + return () => ({ + ...target.spanContext(), + traceState: createTraceState("k=v"), + }); + const value = Reflect.get(target, key, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); + vi.spyOn(console, "error").mockImplementation(() => {}); + shared[customizersKey] = [ + { + onSpanExport(data: Data) { + expect(data.spanContext.traceState).toBe("k=v"); + if (fail) throw new Error("PRIVATE"); + data.spanContext.traceState = "x=y"; + data.spanContext.isRemote = true; + return data; + }, + }, + ]; + await send([withTraceState]); + expect(received()).toHaveLength(1); + expect(received()[0].traceState).toBe("k=v"); }, ); + + test("stripped spans do not split ResourceSpans for later spans", async () => { + const [child, root] = await fixture(); + const third = new Proxy(child, { + get(target, key) { + if (key === "name") return "third"; + const value = Reflect.get(target, key, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); + vi.spyOn(console, "error").mockImplementation(() => {}); + shared[customizersKey] = [ + { + onSpanExport(data: Data) { + if (data.name === "PRIVATE-root") throw new Error("PRIVATE"); + return data; + }, + }, + ]; + await send([child, root, third]); + expect(received()).toHaveLength(3); + // One entry for the stripped span's empty resource, one shared by the rest. + expect(bodies[0].resourceSpans).toHaveLength(2); + }); + + test("drops malformed spans with a safe diagnostic and exports the rest", async () => { + const [child, root] = await fixture(); + const malformed = new Proxy(root, { + get(target, key) { + if (key === "startTime") return undefined; + const value = Reflect.get(target, key, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + shared[customizersKey] = [{ onSpanExport: (data: Data) => data }]; + await send([child, malformed]); + expect(received().map((span) => span.name)).toEqual(["PRIVATE-child"]); + expect(errors).toHaveBeenCalledTimes(1); + expect(JSON.stringify(errors.mock.calls)).not.toContain("PRIVATE"); + }); + + test("does not customize unsampled recorded spans", async () => { + const hook = vi.fn((data: Data) => data); + shared[customizersKey] = [{ onSpanExport: hook }]; + const processor = new BraintrustSpanProcessor({ + apiKey: "test", + apiUrl, + parent: "project_name:test", + }); + const provider = createTracerProvider(BasicTracerProvider, [processor], { + sampler: { + shouldSample: () => ({ decision: SamplingDecision.RECORD }), + toString: () => "RecordOnly", + }, + }); + try { + provider.getTracer("unsampled").startSpan("recorded").end(); + await provider.forceFlush(); + } finally { + await provider.shutdown(); + } + expect(hook).not.toHaveBeenCalled(); + expect(received()).toEqual([]); + }); + + test("keeps non-finite numeric attributes instead of stripping the span", async () => { + const [span] = await fixture(); + const nonFinite = new Proxy(span, { + get(target, key) { + if (key === "attributes") + return { nan: NaN, inf: Infinity, list: [-Infinity] }; + const value = Reflect.get(target, key, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const hook = vi.fn((data: Data) => { + expect(Number.isNaN(data.attributes.nan)).toBe(true); + expect(data.attributes.inf).toBe(Infinity); + return data; + }); + shared[customizersKey] = [{ onSpanExport: hook }]; + await send([nonFinite]); + expect(hook).toHaveBeenCalledTimes(1); + expect(errors).not.toHaveBeenCalled(); + const [exported] = received(); + expect(exported.name).toBe(span.name); + expect(JSON.stringify(exported)).not.toContain("customizer_error"); + expect(exported.attributes.map((attr: Data) => attr.key)).toEqual( + expect.arrayContaining(["nan", "inf", "list"]), + ); + }); + + test("passes spans through when braintrust lacks the customizer runtime", async () => { + const runtimeKey = Symbol.for("braintrust.spanCustomizerRuntime"); + const runtime = shared[runtimeKey]; + expect(runtime).toBeDefined(); + expect("_internalSpanCustomizer" in braintrust).toBe(false); + const hook = vi.fn((data: Data) => data); + shared[customizersKey] = [{ onSpanExport: hook }]; + delete shared[runtimeKey]; + try { + await send(await fixture()); + } finally { + shared[runtimeKey] = runtime; + } + expect(hook).not.toHaveBeenCalled(); + expect(received().map((span) => span.name)).toEqual([ + "PRIVATE-child", + "PRIVATE-root", + ]); + }); + + test("spans sharing a resource stay in one ResourceSpans entry", async () => { + shared[customizersKey] = [{ onSpanExport: (data: Data) => data }]; + await send(await fixture()); + expect(received()).toHaveLength(2); + expect(bodies).toHaveLength(1); + expect(bodies[0].resourceSpans).toHaveLength(1); + }); + + test("a failing later customizer cannot mutate another span's resource", async () => { + const spans = await fixture(); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + shared[customizersKey] = [ + { onSpanExport: (data: Data) => data }, + { + onSpanExport(data: Data) { + if (data.name === "PRIVATE-root") { + data.resource.attributes.secret = "CUSTOMIZER-SECRET"; + throw new Error("CUSTOMIZER-SECRET"); + } + return data; + }, + }, + ]; + + await send(spans); + + const output = received(); + expect( + output.find((span) => span.spanId === spans[0].spanContext().spanId), + ).toHaveProperty("name", "PRIVATE-child"); + const failed = output.find( + (span) => span.spanId === spans[1].spanContext().spanId, + )!; + expect(failed.name).toBe(""); + expect(failed.attributes).toContainEqual({ + key: "braintrust.context_json", + value: { stringValue: '{"customizer_error":true}' }, + }); + expect(errors).toHaveBeenCalledTimes(1); + expect(spans[0].resource.attributes.secret).toBeUndefined(); + expect(JSON.stringify(bodies)).not.toContain("CUSTOMIZER-SECRET"); + }); + + test("customFilter sees span origin context before customization", async () => { + const contexts: unknown[] = []; + shared[customizersKey] = [{ onSpanExport: (data: Data) => data }]; + const processor = new BraintrustSpanProcessor({ + apiKey: "test", + apiUrl, + parent: "project_name:test", + filterAISpans: true, + customFilter: (span) => { + contexts.push(span.attributes["braintrust.context_json"]); + return true; + }, + }); + const provider = createTracerProvider(BasicTracerProvider, [processor]); + try { + provider.getTracer("consumer").startSpan("plain").end(); + await provider.forceFlush(); + } finally { + await provider.shutdown(); + } + expect(contexts).toHaveLength(1); + expect(JSON.parse(contexts[0] as string)).toHaveProperty("span_origin"); + expect(received().map((span) => span.name)).toEqual(["plain"]); + }); }); diff --git a/js/src/id-gen.ts b/js/src/id-gen.ts index 54610d587..67925916a 100644 --- a/js/src/id-gen.ts +++ b/js/src/id-gen.ts @@ -101,18 +101,6 @@ function parseEnvBool(name: string): boolean { ); } -/** - * Whether OpenTelemetry compat mode is active, either via BRAINTRUST_OTEL_COMPAT - * or because `@braintrust/otel`'s `setupOtelCompat()` installed its globals. - */ -export function isOtelCompatMode(): boolean { - return ( - parseEnvBool("BRAINTRUST_OTEL_COMPAT") || - globalThis.BRAINTRUST_CONTEXT_MANAGER !== undefined || - globalThis.BRAINTRUST_ID_GENERATOR !== undefined - ); -} - let _warnedLegacyUuidConflict = false; /** diff --git a/js/src/instrumentation/README.md b/js/src/instrumentation/README.md index 389579075..a8be26ee3 100644 --- a/js/src/instrumentation/README.md +++ b/js/src/instrumentation/README.md @@ -85,10 +85,10 @@ Rebuild bundles transformed with the previous SDK when upgrading; the old combin ## Export Customizers Configure `spanCustomizers` through the standalone instrumentation entrypoint -before importing the main SDK, which enables instrumentation during platform -initialization. Use a bootstrap module before any auto-instrumentation preload -that initializes the SDK. Static imports of the main SDK are hoisted; use a -dynamic import after configuration: +before importing `braintrust` or `@braintrust/otel`. The main SDK enables +instrumentation during platform initialization. Use a bootstrap module before +any auto-instrumentation preload that initializes the SDK. Static imports are +hoisted; use a dynamic import after configuration: ```ts import { configureInstrumentation } from "braintrust/instrumentation"; @@ -121,17 +121,18 @@ Dataset rows and feedback records are not customized. Callbacks run synchronously in registration order. Mutate and return the record, or return a replacement plain object for the next callback. If a callback throws or returns an invalid value (including a promise), the SDK logs a safe error, -stops the callback chain, and drops that outgoing record. Diagnostics do not -include the exception message, stack, or span payload, and are throttled to -the first failure plus at most one report per minute with a suppressed count. Promise rejections are -consumed without awaiting the result. Handle recovery inside the callback if -export should continue, and do not mutate the record after returning. -Export retries reuse the transformed record or drop result without invoking -callbacks or logging the failure again. Unrelated records and future records -from the same span are evaluated independently; previously exported records -cannot be retracted. Dropping a span's first record, which carries -`span_attributes` and `created`, while later merge records succeed can leave a -partial row without a name or type. Configuration is shared across SDK bundles. +stops the callback chain, and exports only original identity, routing, and +duration timestamps plus `context: { customizer_error: true }`. No attachments +from that record upload. Diagnostics do not include the exception message, +stack, or span payload, and are logged at most once per JavaScript global +environment. Promise rejections are consumed without awaiting the result. +Handle recovery inside the callback to retain span content, and do not mutate +the record after returning. Export retries reuse the transformed or stripped +record without invoking callbacks or logging the failure again. Stripped +records bypass downstream masking and remain separate from other updates +with the same ID in ordered export requests. Unrelated records and future +records from the same span are evaluated independently; previously exported +records cannot be retracted. Configuration is shared across SDK bundles. The SDK restores these fields after every callback, including removing injected fields that were absent from the original record: @@ -158,13 +159,79 @@ that guarantee. Customizers receive only the outgoing record, not a live span or provider instrumentation context. -Span customizers are not yet supported with OpenTelemetry compat mode -(`BRAINTRUST_OTEL_COMPAT` or `setupOtelCompat()`). Registering a non-empty list -while compat mode is active logs an error and leaves the previous registration -unchanged. Calling `setupOtelCompat()` after registering customizers logs an error -and continues enabling compat mode without clearing the registered list. Each -explicit registration or setup attempt logs once, not once per span or export. -Clearing customizers is always allowed and silent. +### OpenTelemetry spans + +The same registration applies to spans exported through `@braintrust/otel`'s +`BraintrustSpanProcessor` or `BraintrustExporter`, including compatibility mode +(`BRAINTRUST_OTEL_COMPAT` or `setupOtelCompat()`). Compatibility mode is not +required to customize OTel exports. Native spans still use the native record +shape above; OTel hooks receive an isolated plain snapshot, not a live +`ReadableSpan` or a converted native record: + +- `name`, `kind`, `attributes`, `status: { code, message? }`, `events`, `links`. +- `spanContext: { traceId, spanId, traceFlags, traceState?, isRemote? }`, + `parentSpanId`, `parentSpanContext`, `startTime`, `endTime`, `duration`. + `spanContext` is an object, not a method. Times are OTel + `[seconds, nanoseconds]` arrays; trace state is a string. +- `resource: { attributes, schemaUrl? }`, `instrumentationScope: +{ name, version?, schemaUrl?, attributes? }`, and dropped attribute/event/link + counts. OTel 1.x's instrumentation library is exposed as `instrumentationScope`. +- Events contain `name`, `time`, optional attributes and dropped count; links + contain a plain `context` (the same shape as `spanContext`), optional attributes + and dropped count. + +Hooks run after AI filtering and SDK origin enrichment, before OTLP +serialization. Pending resource attributes are awaited before taking the +snapshot. Plain nested containers are copied, so hook mutations do not affect +the original span or another exporter. Replacement objects are not merged with +the original: omitted content is removed, with empty OTLP defaults where +required. Original identity, parent relationships, timestamps and +`braintrust.parent` routing are restored after each callback. + +For example, register one synchronous hook that handles both payload shapes: + +```ts +import { configureInstrumentation } from "braintrust/instrumentation"; + +configureInstrumentation({ + spanCustomizers: [ + { + onSpanExport(data) { + if ("attributes" in data) { + const attributes = data.attributes as Record; + delete attributes["braintrust.input_json"]; + delete attributes["braintrust.output_json"]; + delete attributes["gen_ai.prompt"]; + delete attributes["gen_ai.completion"]; + data.events = []; + } else { + delete data.input; + delete data.output; + } + return data; + }, + }, + ], +}); + +// Import the application and its tracing setup only after registration. +await import("./app.js"); +``` + +This example removes common prompt/output attributes and events, not every +possible sensitive field. Adjust redaction to the instrumentation you use, +including resource/scope metadata, links and status descriptions as needed. + +Thrown exceptions, promises, non-plain returns and invalid OTel serialization +shapes stop the callback chain and fail closed. The failed span keeps only +original identity, timestamps, routing and `braintrust.context_json` exactly +`{"customizer_error":true}`. Names, status descriptions, events, links, +resource/scope metadata and original context/origin are removed. Malformed spans +that cannot be safely adapted are dropped instead. The safe once-only diagnostic +is shared with the native pipeline across ESM/CJS bundles. Composing the +Braintrust processor and exporter does not invoke hooks twice; retries of the +prepared transport data do not rerun hooks. Submitting the original span again +is a new export. ## Testing diff --git a/js/src/instrumentation/config.ts b/js/src/instrumentation/config.ts index 93d08d96c..ef6b7605f 100644 --- a/js/src/instrumentation/config.ts +++ b/js/src/instrumentation/config.ts @@ -2,9 +2,10 @@ export type SpanExportData = Record; export interface SpanCustomizer { /** - * Customize each outgoing native SDK span record, including manual, instrumented, - * logger, and experiment spans, after lazy values resolve and before JSON - * serialization. Records are incremental and may not contain every span field. + * Customize outgoing native SDK records and Braintrust OpenTelemetry snapshots, + * including manual, instrumented, logger, and experiment spans, before transport + * serialization. Native records are incremental; OTel snapshots contain + * attributes, events, links, resource, instrumentationScope and spanContext. * Dataset rows and feedback records are not customized. * * Callbacks are synchronous. Add, change, or delete payload fields, then return @@ -12,8 +13,9 @@ export interface SpanCustomizer { * Attachment objects; attachment processing and serialization happen later. * Throwing or returning an invalid value (including a promise) stops the chain * and exports only original identity, routing, and duration timestamps, plus - * context: { customizer_error: true }. No attachments from that record upload. - * A safe error is logged at most once per JavaScript global environment. + * context: { customizer_error: true } (OTel: braintrust.context_json string + * attribute). No attachments from that record upload. A safe error is logged + * at most once per JavaScript global environment across both pipelines. * Later records are evaluated independently; already exported records cannot * be retracted. Handle recovery inside the callback to retain span content. */ @@ -70,11 +72,10 @@ export interface InstrumentationConfig { integrations?: InstrumentationIntegrationsConfig; /** - * Customizers for all native SDK spans, in callback execution order. + * Customizers for all native SDK and Braintrust OpenTelemetry spans, in order. + * Native hooks receive span records; OTel hooks receive plain snapshots with + * attributes, events, links, resource, instrumentationScope and spanContext. * Configure before initializing the SDK or enabling instrumentation. - * Not yet supported with OpenTelemetry compat mode; registering a non-empty - * list while compat mode is active logs an error and leaves the previous - * registration unchanged. Clearing customizers is always allowed and silent. */ spanCustomizers?: readonly SpanCustomizer[]; } diff --git a/js/src/span-customizer.test.ts b/js/src/span-customizer.test.ts index f1fd32cf3..954d46c7a 100644 --- a/js/src/span-customizer.test.ts +++ b/js/src/span-customizer.test.ts @@ -1461,7 +1461,9 @@ describe("onSpanExport", () => { }); describe("OpenTelemetry compat mode", () => { - const customizer = { onSpanExport: (data: SpanExportData) => data }; + const customizer = { + onSpanExport: (data: SpanExportData) => ({ ...data, output: "redacted" }), + }; beforeEach(() => registry.disable()); @@ -1480,47 +1482,26 @@ describe("OpenTelemetry compat mode", () => { globalThis.BRAINTRUST_CONTEXT_MANAGER = class {} as never; }, ], - ])("rejects registering customizers under %s", (_name, enableCompat) => { + ])("registers and clears customizers under %s", (_name, enableCompat) => { const errorLog = vi.spyOn(console, "error").mockImplementation(() => {}); enableCompat(); configureInstrumentation({ spanCustomizers: [customizer] }); - expect(errorLog).toHaveBeenCalledTimes(1); - const record = { id: "row", output: "original" }; - expect(customizeSpanExport(record)).toBe(record); - expect(customizeSpanExport({ ...record, id: "row-2" })).toEqual({ - ...record, - id: "row-2", + expect(customizeSpanExport({ id: "row", output: "private" })).toEqual({ + id: "row", + output: "redacted", }); - expect(errorLog).toHaveBeenCalledTimes(1); - - configureInstrumentation({ spanCustomizers: [customizer] }); - expect(errorLog).toHaveBeenCalledTimes(2); - - configureInstrumentation({ spanCustomizers: [] }); - configureInstrumentation({ spanCustomizers: undefined }); - expect(errorLog).toHaveBeenCalledTimes(2); - }); - - test("keeps the previous registration after rejecting a replacement", () => { - const errorLog = vi.spyOn(console, "error").mockImplementation(() => {}); configureInstrumentation({ spanCustomizers: [ - { - onSpanExport: (data) => ({ ...data, output: "previous customizer" }), - }, + { onSpanExport: (data) => ({ ...data, output: "replacement" }) }, ], }); - vi.stubEnv("BRAINTRUST_OTEL_COMPAT", "1"); - - configureInstrumentation({ spanCustomizers: [customizer] }); - expect(errorLog).toHaveBeenCalledTimes(1); - expect(customizeSpanExport({ output: "original" })).toEqual({ - output: "previous customizer", + expect(customizeSpanExport({ output: "private" })).toEqual({ + output: "replacement", }); - - configureInstrumentation({ spanCustomizers: undefined }); + configureInstrumentation({ spanCustomizers: [] }); const record = { output: "original" }; expect(customizeSpanExport(record)).toBe(record); - expect(errorLog).toHaveBeenCalledTimes(1); + configureInstrumentation({ spanCustomizers: undefined }); + expect(errorLog).not.toHaveBeenCalled(); }); }); diff --git a/js/src/span-customizer.ts b/js/src/span-customizer.ts index 837e8a789..a86eda4df 100644 --- a/js/src/span-customizer.ts +++ b/js/src/span-customizer.ts @@ -10,7 +10,6 @@ import { } from "../util/db_fields"; import { mergeRowBatch } from "../util/merge_row_batch"; import { isPromiseLike } from "../util/type_util"; -import { isOtelCompatMode } from "./id-gen"; import type { SpanCustomizer, SpanExportData } from "./instrumentation/config"; // Configuration can precede platform initialization and must be shared across @@ -20,10 +19,17 @@ const CUSTOMIZER_FAILURE_LOGGED_KEY = Symbol.for( "braintrust.spanCustomizerFailureLogged", ); const STRIPPED_RECORDS_KEY = Symbol.for("braintrust.strippedSpanRecords"); +// Integrations with a different export shape find the shared policy through a +// global symbol rather than a public export. The first loaded SDK copy wins; all +// policy state is global, so copies are interchangeable. +const SPAN_CUSTOMIZER_RUNTIME_KEY = Symbol.for( + "braintrust.spanCustomizerRuntime", +); const shared: typeof globalThis & { [SPAN_CUSTOMIZERS_KEY]?: readonly SpanCustomizer[]; [CUSTOMIZER_FAILURE_LOGGED_KEY]?: boolean; [STRIPPED_RECORDS_KEY]?: WeakSet; + [SPAN_CUSTOMIZER_RUNTIME_KEY]?: object; } = globalThis; const PROTECTED_FIELDS = new Set([ @@ -216,14 +222,6 @@ export function mergeCustomizedSpanRecords< export function setSpanCustomizers( customizers: readonly SpanCustomizer[] | undefined, ): void { - if (customizers?.length && isOtelCompatMode()) { - // Keep in sync with setupOtelCompat() in @braintrust/otel. - // eslint-disable-next-line no-restricted-properties -- intentional always-visible configuration error. - console.error( - "Braintrust span customizers are not supported with OpenTelemetry compat mode yet.", - ); - return; - } shared[SPAN_CUSTOMIZERS_KEY] = customizers; } @@ -237,29 +235,44 @@ export function customizeSpanExport( // user content for downstream masking to replace. if (!customizers?.length || shared[STRIPPED_RECORDS_KEY]?.has(data)) return data; + const protectedFields: SpanExportData = {}; + return runSpanCustomizers(data, customizers, { + snapshot(original) { + const snapshot = copyPlainRecord(original); + for (const key of PROTECTED_FIELDS) { + if (Object.prototype.hasOwnProperty.call(snapshot, key)) { + protectedFields[key] = copyProtocolValue(snapshot[key]); + } + } + return snapshot; + }, + restore: (result) => restoreProtocolFields(result, protectedFields), + strip: stripFailedRecord, + }); +} - const original = data; - let protectedFields: SpanExportData | undefined; +interface SpanExportAdapter { + snapshot(data: SpanExportData): SpanExportData; + restore(data: SpanExportData): SpanExportData; + strip(data: SpanExportData): SpanExportData; +} +function runSpanCustomizers( + original: SpanExportData, + customizers: readonly SpanCustomizer[], + adapter: SpanExportAdapter, +): SpanExportData { + let data = original; + let initialized = false; for (const customizer of customizers) { try { if (!customizer.onSpanExport) continue; - if (!protectedFields) { - // Records share nested payload objects with the local span cache, so - // hooks get their own plain containers to mutate. - data = copyPlainRecord(data); - protectedFields = {}; - for (const key of PROTECTED_FIELDS) { - if (Object.prototype.hasOwnProperty.call(data, key)) { - protectedFields[key] = copyProtocolValue(data[key]); - } - } + if (!initialized) { + data = adapter.snapshot(original); + initialized = true; } - const result: unknown = customizer.onSpanExport(data); if (isPromiseLike(result)) { - // Hooks are synchronous, but accidental async hooks must not leak an - // unhandled rejection or replace the record with a promise. void Promise.resolve(result).catch(() => {}); throw new TypeError("Span export customizers must be synchronous"); } @@ -268,12 +281,26 @@ export function customizeSpanExport( "Span export customizers must return a plain object", ); } - // Always copy: hooks may freeze their input or return a frozen record. - data = restoreProtocolFields(result, protectedFields); + data = adapter.restore(result); } catch { registerCustomizerFailure(); - return stripFailedRecord(original); + return adapter.strip(original); } } return data; } + +shared[SPAN_CUSTOMIZER_RUNTIME_KEY] ??= { + isPlainRecord, + hasCustomizers: () => + !!shared[SPAN_CUSTOMIZERS_KEY]?.some( + (customizer) => customizer.onSpanExport, + ), + customize(data: SpanExportData, adapter: SpanExportAdapter): SpanExportData { + return runSpanCustomizers( + data, + shared[SPAN_CUSTOMIZERS_KEY] ?? [], + adapter, + ); + }, +};