diff --git a/js/CHANGELOG.md b/js/CHANGELOG.md index e6f60a272..8760682c4 100644 --- a/js/CHANGELOG.md +++ b/js/CHANGELOG.md @@ -1,5 +1,11 @@ # braintrust +## Unreleased + +### Minor Changes + +- feat: Run configured `onSpanExport` customizers on incremental instrumentation span records, supporting field mutation, deletion, and replacement before export. + ## 3.32.0 ### Minor Changes diff --git a/js/src/exports.ts b/js/src/exports.ts index 0d4d20b2f..b122a7e56 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -373,6 +373,10 @@ export { braintrustFlueObserver, braintrustFlueInstrumentation, } from "./instrumentation"; -export type { InstrumentationConfig } from "./instrumentation"; +export type { + InstrumentationConfig, + SpanCustomizer, + SpanExportData, +} from "./instrumentation"; export { wrapElevenLabs } from "./wrappers/elevenlabs"; diff --git a/js/src/instrumentation/README.md b/js/src/instrumentation/README.md index 26487ec26..b07fcaaf1 100644 --- a/js/src/instrumentation/README.md +++ b/js/src/instrumentation/README.md @@ -180,6 +180,49 @@ termination, and async context. - Use narrow vendored provider interfaces shared by wrappers and plugins. - Keep enable, disable, subscription, and patching behavior idempotent. +## Export Customizers + +Configure `spanCustomizers` through the standalone instrumentation entrypoint +before importing the main SDK, which enables instrumentation during platform +initialization. Use a bootstrap module before any auto-instrumentation preload +that initializes the SDK. Static imports of the main SDK are hoisted; use a +dynamic import after configuration: + +```ts +import { configureInstrumentation } from "braintrust/instrumentation"; + +configureInstrumentation({ + spanCustomizers: [ + { + onSpanExport(data) { + data.tags = ["reviewed"]; + if ("output" in data) data.output = "[redacted]"; + delete data.error; + return data; + }, + }, + ], +}); + +const { initLogger } = await import("braintrust"); +initLogger({ projectName: "my-project" }); +// Import and use instrumented provider SDKs here. +``` + +`onSpanExport` receives each incremental record from an instrumentation-created +span after lazy values resolve, before attachment processing, merging, masking, +and JSON serialization. It can run before the span ends; fields may be absent. +Ordinary manually created spans, dataset rows, and feedback are not customized. + +Callbacks run synchronously in registration order. Mutate and return the record, +or return a replacement for the next callback. Preserve identity and routing +fields and return JSON-serializable data. Exceptions are swallowed; remaining +customizers and export continue. Export retries reuse the transformed record +without invoking callbacks again. Configuration is shared across SDK bundles. + +Customizers receive only the outgoing record, not a live span or provider +instrumentation context. + ## Testing Test at the narrowest useful layers: diff --git a/js/src/instrumentation/config.ts b/js/src/instrumentation/config.ts index 784ca7b2b..8236a50bd 100644 --- a/js/src/instrumentation/config.ts +++ b/js/src/instrumentation/config.ts @@ -1,3 +1,17 @@ +export type SpanExportData = Record; + +export interface SpanCustomizer { + /** + * Customize an outgoing span record after lazy values resolve, before JSON + * serialization. Records are incremental and may not contain every span field. + * + * Add, change, or delete fields, then return the record or a replacement. + * Preserve identity and routing fields, including id, span_id, root_span_id, + * and span_parents. + */ + onSpanExport?(data: SpanExportData): SpanExportData; +} + export interface InstrumentationIntegrationsConfig { openai?: boolean; anthropic?: boolean; @@ -45,6 +59,12 @@ export interface InstrumentationConfig { * Set to false to disable instrumentation for that SDK. */ integrations?: InstrumentationIntegrationsConfig; + + /** + * Instrumentation-wide customizers, in callback execution order. + * Configure before instrumentation is enabled. + */ + spanCustomizers?: readonly SpanCustomizer[]; } const envIntegrationAliases: Record< diff --git a/js/src/instrumentation/index.ts b/js/src/instrumentation/index.ts index a1dbd990c..833786cc5 100644 --- a/js/src/instrumentation/index.ts +++ b/js/src/instrumentation/index.ts @@ -45,3 +45,4 @@ export { // Configuration API export { configureInstrumentation } from "./registry"; export type { InstrumentationConfig } from "./registry"; +export type { SpanCustomizer, SpanExportData } from "./config"; diff --git a/js/src/instrumentation/registry.ts b/js/src/instrumentation/registry.ts index 0ed195ae8..1ed665a0a 100644 --- a/js/src/instrumentation/registry.ts +++ b/js/src/instrumentation/registry.ts @@ -13,6 +13,7 @@ import { type InstrumentationConfig, } from "./config"; import { GLOBAL_INSTRUMENTATION_HOOKS_PROTOCOL_VERSION } from "../global-instrumentation-hooks"; +import { setSpanCustomizers } from "../span-customizer"; export type { InstrumentationConfig } from "./config"; @@ -62,6 +63,9 @@ class PluginRegistry { return; } this.config = { ...this.config, ...config }; + if ("spanCustomizers" in config) { + setSpanCustomizers(config.spanCustomizers); + } } /** diff --git a/js/src/logger.ts b/js/src/logger.ts index 5db55e25d..3b10807fb 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -207,6 +207,7 @@ import { mergeSpanOriginContext, type SpanOriginEnvironment, } from "./span-origin"; +import { customizeSpanExport } from "./span-customizer"; // Manual type definition for inline attachments (not in generated_types) const InlineAttachmentReferenceSchema = z.object({ @@ -8140,6 +8141,7 @@ export class SpanImpl implements Span { private isMerge: boolean; private loggedEndTime: number | undefined; + private readonly isInstrumented: boolean; private propagatedEvent: StartSpanEventArgs | undefined; // For internal use only. @@ -8180,6 +8182,8 @@ export class SpanImpl implements Span { const instrumentationName = getSpanInstrumentationName(args) ?? INSTRUMENTATION_NAMES.BRAINTRUST_JS_LOGGER; + this.isInstrumented = + instrumentationName !== INSTRUMENTATION_NAMES.BRAINTRUST_JS_LOGGER; const spanAttributes = args.spanAttributes ?? {}; const rawEvent = args.event ?? {}; @@ -8347,21 +8351,28 @@ export class SpanImpl implements Span { ); } - const computeRecord = async () => ({ - ...partialRecord, - ...Object.fromEntries( - await Promise.all( - Object.entries(lazyInternalData).map(async ([key, value]) => [ - key, - await value.get(), - ]), + const computeRecord = async () => { + const record = { + ...partialRecord, + ...Object.fromEntries( + await Promise.all( + Object.entries(lazyInternalData).map(async ([key, value]) => [ + key, + await value.get(), + ]), + ), ), - ), - ...new SpanComponentsV3({ - object_type: this.parentObjectType, - object_id: await this.parentObjectId.get(), - }).objectIdFields(), - }); + ...new SpanComponentsV3({ + object_type: this.parentObjectType, + object_id: await this.parentObjectId.get(), + }).objectIdFields(), + }; + // Customize inside the memoized lazy value, before attachment processing, + // merging, and masking. Retries reuse the already-customized record. + return this.isInstrumented + ? (customizeSpanExport(record) as BackgroundLogEvent) + : record; + }; this._state.bgLogger().log([new LazyValue(computeRecord)]); } diff --git a/js/src/span-customizer.test.ts b/js/src/span-customizer.test.ts new file mode 100644 index 000000000..21df7c70c --- /dev/null +++ b/js/src/span-customizer.test.ts @@ -0,0 +1,190 @@ +import { + afterEach, + beforeEach, + describe, + expect, + expectTypeOf, + test, +} from "vitest"; +import { + _exportsForTestingOnly, + initLogger, + type TestBackgroundLogger, +} from "./logger"; +import { configureInstrumentation, registry } from "./instrumentation/registry"; +import { configureNode } from "./node/config"; +import { + INSTRUMENTATION_NAMES, + withSpanInstrumentationName, +} from "./span-origin"; +import type { SpanCustomizer, SpanExportData } from "./exports"; + +configureNode(); + +test("customizers expose only the outgoing-record export hook", () => { + expectTypeOf().toEqualTypeOf<{ + onSpanExport?(data: SpanExportData): SpanExportData; + }>(); +}); + +describe("onSpanExport", () => { + let memoryLogger: TestBackgroundLogger; + + beforeEach(async () => { + registry.disable(); + await _exportsForTestingOnly.simulateLoginForTests(); + memoryLogger = _exportsForTestingOnly.useTestBackgroundLogger(); + }); + + afterEach(() => { + configureInstrumentation({ spanCustomizers: [] }); + _exportsForTestingOnly.clearTestBackgroundLogger(); + }); + + function startInstrumentedSpan() { + return initLogger({ + projectName: "customizer-project", + projectId: "customizer-project", + }).startSpan( + withSpanInstrumentationName( + { name: "provider.call" }, + INSTRUMENTATION_NAMES.OPENAI, + ), + ); + } + + test("adds a field to outgoing span records", async () => { + configureInstrumentation({ + spanCustomizers: [ + { + onSpanExport(data) { + data.custom_field = "added"; + return data; + }, + }, + ], + }); + + const span = startInstrumentedSpan(); + span.log({ output: "result" }); + span.end(); + + const events = await memoryLogger.drain(); + expect(events).toEqual([ + expect.objectContaining({ + id: span.id, + project_id: "customizer-project", + output: "result", + custom_field: "added", + }), + ]); + }); + + test("alters an existing field in outgoing span records", async () => { + configureInstrumentation({ + spanCustomizers: [ + { + onSpanExport(data) { + if ("output" in data) data.output = "[redacted]"; + return data; + }, + }, + ], + }); + + const span = startInstrumentedSpan(); + span.log({ output: "sensitive response" }); + span.end(); + + expect(await memoryLogger.drain()).toEqual([ + expect.objectContaining({ id: span.id, output: "[redacted]" }), + ]); + }); + + test("deletes a field from outgoing span records", async () => { + configureInstrumentation({ + spanCustomizers: [ + { + onSpanExport(data) { + delete data.error; + return data; + }, + }, + ], + }); + + const span = startInstrumentedSpan(); + span.log({ error: "sensitive error", output: "safe response" }); + span.end(); + + const events = await memoryLogger.drain(); + expect(events).toEqual([ + expect.objectContaining({ id: span.id, output: "safe response" }), + ]); + expect(events[0]).not.toHaveProperty("error"); + }); + + test("passes replacement records through later customizers despite errors", async () => { + configureInstrumentation({ + spanCustomizers: [ + { + onSpanExport(data) { + return "output" in data ? { ...data, output: "replacement" } : data; + }, + }, + { + onSpanExport() { + throw new Error("customizer failed"); + }, + }, + { + onSpanExport(data) { + if (typeof data.output === "string") { + data.output = data.output.toUpperCase(); + } + return data; + }, + }, + ], + }); + + const span = startInstrumentedSpan(); + span.log({ output: "original" }); + span.end(); + + expect(await memoryLogger.drain()).toEqual([ + expect.objectContaining({ + id: span.id, + output: "REPLACEMENT", + metrics: expect.objectContaining({ end: expect.any(Number) }), + }), + ]); + }); + + test("does not customize manually created spans", async () => { + configureInstrumentation({ + spanCustomizers: [ + { + onSpanExport(data) { + data.tags = ["customized"]; + return data; + }, + }, + ], + }); + + const instrumented = startInstrumentedSpan(); + const manual = instrumented.startSpan({ name: "manual child" }); + manual.log({ output: "manual result" }); + manual.end(); + instrumented.end(); + + const events = await memoryLogger.drain(); + expect(events.find((event) => event.id === instrumented.id)).toMatchObject({ + tags: ["customized"], + }); + const manualEvent = events.find((event) => event.id === manual.id); + expect(manualEvent).toMatchObject({ output: "manual result" }); + expect(manualEvent).not.toHaveProperty("tags"); + }); +}); diff --git a/js/src/span-customizer.ts b/js/src/span-customizer.ts new file mode 100644 index 000000000..81925c9f7 --- /dev/null +++ b/js/src/span-customizer.ts @@ -0,0 +1,30 @@ +import type { SpanCustomizer, SpanExportData } from "./instrumentation/config"; + +// Configuration can precede platform initialization and must be shared across +// SDK bundles without importing the provider plugin registry into the logger. +const SPAN_CUSTOMIZERS_KEY = Symbol.for("braintrust.spanCustomizers"); +const shared: typeof globalThis & { + [SPAN_CUSTOMIZERS_KEY]?: readonly SpanCustomizer[]; +} = globalThis; + +export function setSpanCustomizers( + customizers: readonly SpanCustomizer[] | undefined, +): void { + shared[SPAN_CUSTOMIZERS_KEY] = customizers; +} + +export function customizeSpanExport(data: SpanExportData): SpanExportData { + const customizers = shared[SPAN_CUSTOMIZERS_KEY]; + if (!customizers) return data; + + for (const customizer of customizers) { + try { + if (customizer.onSpanExport) { + data = customizer.onSpanExport(data); + } + } catch { + // Customization must not prevent export or later customizers from running. + } + } + return data; +}