From 6331b119e897a8baa222aa64290c2aebe480a3c8 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Thu, 17 Sep 2026 14:11:26 +0000 Subject: [PATCH 1/2] feat: Fix token metrics with AI SDK middleware --- .../ai-sdk-v5-auto-hook.span-tree.json | 12 +- .../ai-sdk-v5-auto-hook.span-tree.txt | 12 +- .../ai-sdk-v5-latest-auto-hook.span-tree.json | 12 +- .../ai-sdk-v5-latest-auto-hook.span-tree.txt | 12 +- .../ai-sdk-v5-latest-wrapped.span-tree.json | 325 ++++++++++++- .../ai-sdk-v5-latest-wrapped.span-tree.txt | 304 +++++++++++- .../ai-sdk-v5-wrapped.span-tree.json | 313 +++++++++++- .../ai-sdk-v5-wrapped.span-tree.txt | 292 ++++++++++- .../ai-sdk-v6-latest-wrapped.span-tree.json | 404 ++++++++++++++++ .../ai-sdk-v6-latest-wrapped.span-tree.txt | 383 +++++++++++++++ .../ai-sdk-v6-wrapped.span-tree.json | 388 +++++++++++++++ .../ai-sdk-v6-wrapped.span-tree.txt | 367 ++++++++++++++ .../ai-sdk-v7-auto-hook.span-tree.json | 444 +++++++++++++++++ .../ai-sdk-v7-auto-hook.span-tree.txt | 419 ++++++++++++++++ .../ai-sdk-v7-explicit.span-tree.json | 444 +++++++++++++++++ .../ai-sdk-v7-explicit.span-tree.txt | 419 ++++++++++++++++ .../ai-sdk-v7-latest-auto-hook.span-tree.json | 457 ++++++++++++++++++ .../ai-sdk-v7-latest-auto-hook.span-tree.txt | 432 +++++++++++++++++ .../ai-sdk-v7-latest-explicit.span-tree.json | 457 ++++++++++++++++++ .../ai-sdk-v7-latest-explicit.span-tree.txt | 432 +++++++++++++++++ .../scenario.ai-sdk-v7-explicit.mjs | 1 + .../scenario.ai-sdk-v7.mjs | 1 + .../scenario.ai-sdk-v7.ts | 1 + .../ai-sdk-instrumentation/scenario.impl.mjs | 65 ++- .../plugins/ai-sdk-channels.ts | 27 ++ .../instrumentation/plugins/ai-sdk-metrics.ts | 205 ++++++++ .../plugins/ai-sdk-plugin.streaming.test.ts | 208 ++++++++ .../plugins/ai-sdk-plugin.test.ts | 189 ++++---- .../instrumentation/plugins/ai-sdk-plugin.ts | 433 ++++++++--------- js/src/vendor-sdk-types/ai-sdk-common.ts | 4 +- js/src/wrappers/ai-sdk/README.md | 15 + js/src/wrappers/ai-sdk/ai-sdk.ts | 64 +++ .../ai-sdk/deprecated/BraintrustMiddleware.ts | 115 +---- 33 files changed, 7186 insertions(+), 470 deletions(-) create mode 100644 js/src/instrumentation/plugins/ai-sdk-metrics.ts diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-auto-hook.span-tree.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-auto-hook.span-tree.json index a0959feb2..e96263a30 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-auto-hook.span-tree.json +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-auto-hook.span-tree.json @@ -2873,8 +2873,8 @@ "completion_tokens": 7, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 4516, - "prompt_tokens": 3, - "tokens": 10 + "prompt_tokens": 4519, + "tokens": 4526 } } ], @@ -3121,8 +3121,8 @@ "completion_tokens": 7, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 4516, - "prompt_tokens": 3, - "tokens": 10 + "prompt_tokens": 4519, + "tokens": 4526 } } ], @@ -3366,8 +3366,8 @@ "completion_tokens": 7, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 4516, - "prompt_tokens": 3, - "tokens": 10 + "prompt_tokens": 4519, + "tokens": 4526 } } ], diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-auto-hook.span-tree.txt b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-auto-hook.span-tree.txt index 5f21d4037..06a06189d 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-auto-hook.span-tree.txt +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-auto-hook.span-tree.txt @@ -2893,8 +2893,8 @@ span_tree: │ "completion_tokens": 7, │ "prompt_cache_creation_tokens": 0, │ "prompt_cached_tokens": 4516, - │ "prompt_tokens": 3, - │ "tokens": 10 + │ "prompt_tokens": 4519, + │ "tokens": 4526 │ } ├── ai-sdk-anthropic-cache-operation │ metadata: { @@ -3134,8 +3134,8 @@ span_tree: │ │ "completion_tokens": 7, │ │ "prompt_cache_creation_tokens": 0, │ │ "prompt_cached_tokens": 4516, - │ │ "prompt_tokens": 3, - │ │ "tokens": 10 + │ │ "prompt_tokens": 4519, + │ │ "tokens": 4526 │ │ } │ └── generateText [function] │ input: { @@ -3370,8 +3370,8 @@ span_tree: │ "completion_tokens": 7, │ "prompt_cache_creation_tokens": 0, │ "prompt_cached_tokens": 4516, - │ "prompt_tokens": 3, - │ "tokens": 10 + │ "prompt_tokens": 4519, + │ "tokens": 4526 │ } ├── ai-sdk-deny-output-override-operation │ metadata: { diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-latest-auto-hook.span-tree.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-latest-auto-hook.span-tree.json index 4afc79466..31e6a2487 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-latest-auto-hook.span-tree.json @@ -2955,8 +2955,8 @@ "completion_tokens": 7, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 4516, - "prompt_tokens": 3, - "tokens": 10 + "prompt_tokens": 4519, + "tokens": 4526 } } ], @@ -3203,8 +3203,8 @@ "completion_tokens": 7, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 4516, - "prompt_tokens": 3, - "tokens": 10 + "prompt_tokens": 4519, + "tokens": 4526 } } ], @@ -3448,8 +3448,8 @@ "completion_tokens": 7, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 4516, - "prompt_tokens": 3, - "tokens": 10 + "prompt_tokens": 4519, + "tokens": 4526 } } ], diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-latest-auto-hook.span-tree.txt b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-latest-auto-hook.span-tree.txt index a6779e99d..3d5540e20 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-latest-auto-hook.span-tree.txt @@ -2975,8 +2975,8 @@ span_tree: │ "completion_tokens": 7, │ "prompt_cache_creation_tokens": 0, │ "prompt_cached_tokens": 4516, - │ "prompt_tokens": 3, - │ "tokens": 10 + │ "prompt_tokens": 4519, + │ "tokens": 4526 │ } ├── ai-sdk-anthropic-cache-operation │ metadata: { @@ -3216,8 +3216,8 @@ span_tree: │ │ "completion_tokens": 7, │ │ "prompt_cache_creation_tokens": 0, │ │ "prompt_cached_tokens": 4516, - │ │ "prompt_tokens": 3, - │ │ "tokens": 10 + │ │ "prompt_tokens": 4519, + │ │ "tokens": 4526 │ │ } │ └── generateText [function] │ input: { @@ -3452,8 +3452,8 @@ span_tree: │ "completion_tokens": 7, │ "prompt_cache_creation_tokens": 0, │ "prompt_cached_tokens": 4516, - │ "prompt_tokens": 3, - │ "tokens": 10 + │ "prompt_tokens": 4519, + │ "tokens": 4526 │ } ├── ai-sdk-deny-output-override-operation │ metadata: { diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-latest-wrapped.span-tree.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-latest-wrapped.span-tree.json index d63d8efa9..dc26bd435 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-latest-wrapped.span-tree.json +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-latest-wrapped.span-tree.json @@ -4,6 +4,323 @@ "name": "ai-sdk-instrumentation-root", "type": "task", "children": [ + { + "name": "ai-sdk-direct-model-generate-operation", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with the single token PARIS and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default", + "usage": { + "cacheWriteTokens": 0 + } + } + }, + "response": { + "body": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 18, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 18, + "reasoning_tokens": 0, + "tokens": 21 + } + } + ], + "metadata": { + "operation": "direct-model-generate", + "testRunId": "" + } + }, + { + "name": "ai-sdk-wrapped-model-generate-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with the single token PARIS and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default", + "usage": { + "cacheWriteTokens": 0 + } + } + }, + "response": { + "body": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 18, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 18, + "reasoning_tokens": 0, + "tokens": 21 + } + } + ], + "input": { + "maxOutputTokens": 24, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Reply with the single token PARIS and no punctuation.", + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default", + "usage": { + "cacheWriteTokens": 0 + } + } + }, + "response": { + "body": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default", + "usage": { + "cacheWriteTokens": 0 + } + } + }, + "response": { + "body": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 18, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "warnings": [] + } + ], + "text": "PARIS", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokens": 18, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 18, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "wrapped-model-generate", + "testRunId": "" + } + }, { "name": "ai-sdk-generate-operation", "children": [ @@ -2722,8 +3039,8 @@ "completion_tokens": 7, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 4516, - "prompt_tokens": 3, - "tokens": 10 + "prompt_tokens": 4519, + "tokens": 4526 } } ], @@ -2967,8 +3284,8 @@ "completion_tokens": 7, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 4516, - "prompt_tokens": 3, - "tokens": 10 + "prompt_tokens": 4519, + "tokens": 4526 } } ], diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-latest-wrapped.span-tree.txt b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-latest-wrapped.span-tree.txt index a2df33e25..a55fdc502 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-latest-wrapped.span-tree.txt @@ -5,6 +5,302 @@ span_tree: "scenario": "ai-sdk-instrumentation", "testRunId": "" } + ├── ai-sdk-direct-model-generate-operation + │ metadata: { + │ "operation": "direct-model-generate", + │ "testRunId": "" + │ } + │ └── doGenerate [llm] + │ input: { + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the single token PARIS and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default", + │ "usage": { + │ "cacheWriteTokens": 0 + │ } + │ } + │ }, + │ "response": { + │ "body": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 18, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 18, + │ "reasoning_tokens": 0, + │ "tokens": 21 + │ } + ├── ai-sdk-wrapped-model-generate-operation + │ metadata: { + │ "operation": "wrapped-model-generate", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v2", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Reply with the single token PARIS and no punctuation.", + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default", + │ "usage": { + │ "cacheWriteTokens": 0 + │ } + │ } + │ }, + │ "response": { + │ "body": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default", + │ "usage": { + │ "cacheWriteTokens": 0 + │ } + │ } + │ }, + │ "response": { + │ "body": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 18, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "PARIS", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 18, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 18, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the single token PARIS and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default", + │ "usage": { + │ "cacheWriteTokens": 0 + │ } + │ } + │ }, + │ "response": { + │ "body": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 18, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 18, + │ "reasoning_tokens": 0, + │ "tokens": 21 + │ } ├── ai-sdk-generate-operation │ metadata: { │ "operation": "generate", @@ -2753,8 +3049,8 @@ span_tree: │ │ "completion_tokens": 7, │ │ "prompt_cache_creation_tokens": 0, │ │ "prompt_cached_tokens": 4516, - │ │ "prompt_tokens": 3, - │ │ "tokens": 10 + │ │ "prompt_tokens": 4519, + │ │ "tokens": 4526 │ │ } │ └── generateText [function] │ input: { @@ -2989,8 +3285,8 @@ span_tree: │ "completion_tokens": 7, │ "prompt_cache_creation_tokens": 0, │ "prompt_cached_tokens": 4516, - │ "prompt_tokens": 3, - │ "tokens": 10 + │ "prompt_tokens": 4519, + │ "tokens": 4526 │ } ├── ai-sdk-deny-output-override-operation │ metadata: { diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-wrapped.span-tree.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-wrapped.span-tree.json index 83bb38d73..fbcbb9eba 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-wrapped.span-tree.json +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-wrapped.span-tree.json @@ -4,6 +4,311 @@ "name": "ai-sdk-instrumentation-root", "type": "task", "children": [ + { + "name": "ai-sdk-direct-model-generate-operation", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with the single token PARIS and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 18, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 18, + "reasoning_tokens": 0, + "tokens": 21 + } + } + ], + "metadata": { + "operation": "direct-model-generate", + "testRunId": "" + } + }, + { + "name": "ai-sdk-wrapped-model-generate-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with the single token PARIS and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 18, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 18, + "reasoning_tokens": 0, + "tokens": 21 + } + } + ], + "input": { + "maxOutputTokens": 24, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Reply with the single token PARIS and no punctuation.", + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 18, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "warnings": [] + } + ], + "text": "PARIS", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokens": 18, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 18, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "wrapped-model-generate", + "testRunId": "" + } + }, { "name": "ai-sdk-generate-operation", "children": [ @@ -2649,8 +2954,8 @@ "completion_tokens": 7, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 4516, - "prompt_tokens": 3, - "tokens": 10 + "prompt_tokens": 4519, + "tokens": 4526 } } ], @@ -2894,8 +3199,8 @@ "completion_tokens": 7, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 4516, - "prompt_tokens": 3, - "tokens": 10 + "prompt_tokens": 4519, + "tokens": 4526 } } ], diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-wrapped.span-tree.txt b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-wrapped.span-tree.txt index d03800926..283918fad 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-wrapped.span-tree.txt +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-wrapped.span-tree.txt @@ -5,6 +5,290 @@ span_tree: "scenario": "ai-sdk-instrumentation", "testRunId": "" } + ├── ai-sdk-direct-model-generate-operation + │ metadata: { + │ "operation": "direct-model-generate", + │ "testRunId": "" + │ } + │ └── doGenerate [llm] + │ input: { + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the single token PARIS and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 18, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 18, + │ "reasoning_tokens": 0, + │ "tokens": 21 + │ } + ├── ai-sdk-wrapped-model-generate-operation + │ metadata: { + │ "operation": "wrapped-model-generate", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v2", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Reply with the single token PARIS and no punctuation.", + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 18, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "PARIS", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 18, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 18, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the single token PARIS and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 18, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 18, + │ "reasoning_tokens": 0, + │ "tokens": 21 + │ } ├── ai-sdk-generate-operation │ metadata: { │ "operation": "generate", @@ -2680,8 +2964,8 @@ span_tree: │ │ "completion_tokens": 7, │ │ "prompt_cache_creation_tokens": 0, │ │ "prompt_cached_tokens": 4516, - │ │ "prompt_tokens": 3, - │ │ "tokens": 10 + │ │ "prompt_tokens": 4519, + │ │ "tokens": 4526 │ │ } │ └── generateText [function] │ input: { @@ -2916,8 +3200,8 @@ span_tree: │ "completion_tokens": 7, │ "prompt_cache_creation_tokens": 0, │ "prompt_cached_tokens": 4516, - │ "prompt_tokens": 3, - │ "tokens": 10 + │ "prompt_tokens": 4519, + │ "tokens": 4526 │ } ├── ai-sdk-deny-output-override-operation │ metadata: { diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-latest-wrapped.span-tree.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-latest-wrapped.span-tree.json index a8197eceb..28510a98d 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-latest-wrapped.span-tree.json +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-latest-wrapped.span-tree.json @@ -4,6 +4,410 @@ "name": "ai-sdk-instrumentation-root", "type": "task", "children": [ + { + "name": "ai-sdk-direct-model-generate-operation", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with the single token PARIS and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "cacheWrite": 0, + "noCache": 18, + "total": 18 + }, + "outputTokens": { + "reasoning": 0, + "text": 3, + "total": 3 + }, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 18, + "reasoning_tokens": 0, + "tokens": 21 + } + } + ], + "metadata": { + "operation": "direct-model-generate", + "testRunId": "" + } + }, + { + "name": "ai-sdk-wrapped-model-generate-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with the single token PARIS and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "cacheWrite": 0, + "noCache": 18, + "total": 18 + }, + "outputTokens": { + "reasoning": 0, + "text": 3, + "total": 3 + }, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 18, + "reasoning_tokens": 0, + "tokens": 21 + } + } + ], + "input": { + "maxOutputTokens": 24, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Reply with the single token PARIS and no punctuation.", + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "output": "PARIS", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "model": { + "modelId": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "stepNumber": 0, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "warnings": [] + } + ], + "text": "PARIS", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "wrapped-model-generate", + "testRunId": "" + } + }, { "name": "ai-sdk-generate-operation", "children": [ diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-latest-wrapped.span-tree.txt b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-latest-wrapped.span-tree.txt index 146b3281c..0791e07b1 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-latest-wrapped.span-tree.txt @@ -5,6 +5,389 @@ span_tree: "scenario": "ai-sdk-instrumentation", "testRunId": "" } + ├── ai-sdk-direct-model-generate-operation + │ metadata: { + │ "operation": "direct-model-generate", + │ "testRunId": "" + │ } + │ └── doGenerate [llm] + │ input: { + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the single token PARIS and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "cacheWrite": 0, + │ "noCache": 18, + │ "total": 18 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 3, + │ "total": 3 + │ }, + │ "raw": { + │ "input_tokens": 18, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 18, + │ "reasoning_tokens": 0, + │ "tokens": 21 + │ } + ├── ai-sdk-wrapped-model-generate-operation + │ metadata: { + │ "operation": "wrapped-model-generate", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v3", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Reply with the single token PARIS and no punctuation.", + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "output": "PARIS", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "model": { + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "stepNumber": 0, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 18 + │ }, + │ "inputTokens": 18, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "raw": { + │ "input_tokens": 18, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "PARIS", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 18 + │ }, + │ "inputTokens": 18, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 18 + │ }, + │ "inputTokens": 18, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "raw": { + │ "input_tokens": 18, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the single token PARIS and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "cacheWrite": 0, + │ "noCache": 18, + │ "total": 18 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 3, + │ "total": 3 + │ }, + │ "raw": { + │ "input_tokens": 18, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 18, + │ "reasoning_tokens": 0, + │ "tokens": 21 + │ } ├── ai-sdk-generate-operation │ metadata: { │ "operation": "generate", diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-wrapped.span-tree.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-wrapped.span-tree.json index 296890390..2e7f1a0d2 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-wrapped.span-tree.json +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-wrapped.span-tree.json @@ -4,6 +4,394 @@ "name": "ai-sdk-instrumentation-root", "type": "task", "children": [ + { + "name": "ai-sdk-direct-model-generate-operation", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with the single token PARIS and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 18, + "total": 18 + }, + "outputTokens": { + "reasoning": 0, + "text": 3, + "total": 3 + }, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 18, + "reasoning_tokens": 0, + "tokens": 21 + } + } + ], + "metadata": { + "operation": "direct-model-generate", + "testRunId": "" + } + }, + { + "name": "ai-sdk-wrapped-model-generate-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with the single token PARIS and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 18, + "total": 18 + }, + "outputTokens": { + "reasoning": 0, + "text": 3, + "total": 3 + }, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 18, + "reasoning_tokens": 0, + "tokens": 21 + } + } + ], + "input": { + "maxOutputTokens": 24, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Reply with the single token PARIS and no punctuation.", + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "output": "PARIS", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "warnings": [] + } + ], + "text": "PARIS", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "wrapped-model-generate", + "testRunId": "" + } + }, { "name": "ai-sdk-generate-operation", "children": [ diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-wrapped.span-tree.txt b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-wrapped.span-tree.txt index e78d7a23e..9f81ce3ac 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-wrapped.span-tree.txt +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-wrapped.span-tree.txt @@ -5,6 +5,373 @@ span_tree: "scenario": "ai-sdk-instrumentation", "testRunId": "" } + ├── ai-sdk-direct-model-generate-operation + │ metadata: { + │ "operation": "direct-model-generate", + │ "testRunId": "" + │ } + │ └── doGenerate [llm] + │ input: { + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the single token PARIS and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "noCache": 18, + │ "total": 18 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 3, + │ "total": 3 + │ }, + │ "raw": { + │ "input_tokens": 18, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 18, + │ "reasoning_tokens": 0, + │ "tokens": 21 + │ } + ├── ai-sdk-wrapped-model-generate-operation + │ metadata: { + │ "operation": "wrapped-model-generate", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v3", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Reply with the single token PARIS and no punctuation.", + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "output": "PARIS", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 18 + │ }, + │ "inputTokens": 18, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "raw": { + │ "input_tokens": 18, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "PARIS", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 18 + │ }, + │ "inputTokens": 18, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 18 + │ }, + │ "inputTokens": 18, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "raw": { + │ "input_tokens": 18, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the single token PARIS and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "noCache": 18, + │ "total": 18 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 3, + │ "total": 3 + │ }, + │ "raw": { + │ "input_tokens": 18, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 18, + │ "reasoning_tokens": 0, + │ "tokens": 21 + │ } ├── ai-sdk-generate-operation │ metadata: { │ "operation": "generate", diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-auto-hook.span-tree.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-auto-hook.span-tree.json index 28f97e365..0d15587e4 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-auto-hook.span-tree.json +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-auto-hook.span-tree.json @@ -4,6 +4,450 @@ "name": "ai-sdk-instrumentation-root", "type": "task", "children": [ + { + "name": "ai-sdk-direct-model-generate-operation", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with the single token PARIS and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 18, + "total": 18 + }, + "outputTokens": { + "reasoning": 0, + "text": 3, + "total": 3 + }, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 18, + "reasoning_tokens": 0, + "tokens": 21 + } + } + ], + "metadata": { + "operation": "direct-model-generate", + "testRunId": "" + } + }, + { + "name": "ai-sdk-wrapped-model-generate-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": "Reply with the single token PARIS and no punctuation.", + "role": "user" + } + ], + "model": { + "modelId": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "response": {}, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "totalTokens": 21 + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 18, + "tokens": 21 + } + } + ], + "input": { + "maxOutputTokens": 24, + "maxRetries": 2, + "messages": [ + { + "content": "Reply with the single token PARIS and no punctuation.", + "role": "user" + } + ], + "model": { + "modelId": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "responseMessages": "", + "steps": [ + { + "callId": "", + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "model": { + "modelId": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "runtimeContext": {}, + "stepNumber": 0, + "toolsContext": {}, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "totalTokens": 21 + }, + "warnings": [] + } + ], + "text": "PARIS", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "totalTokens": 21 + }, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "totalTokens": 21 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + }, + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with the single token PARIS and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0, + "toolChoice": { + "type": "auto" + } + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 18, + "total": 18 + }, + "outputTokens": { + "reasoning": 0, + "text": 3, + "total": 3 + }, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 18, + "reasoning_tokens": 0, + "tokens": 21 + } + } + ], + "metadata": { + "operation": "wrapped-model-generate", + "testRunId": "" + } + }, { "name": "ai-sdk-generate-operation", "children": [ diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-auto-hook.span-tree.txt b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-auto-hook.span-tree.txt index 801c1513e..b88bfc755 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-auto-hook.span-tree.txt +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-auto-hook.span-tree.txt @@ -6,6 +6,425 @@ span_tree: "testRunId": "", "workflowVersion": "1.0.8" } + ├── ai-sdk-direct-model-generate-operation + │ metadata: { + │ "operation": "direct-model-generate", + │ "testRunId": "" + │ } + │ └── doGenerate [llm] + │ input: { + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the single token PARIS and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "noCache": 18, + │ "total": 18 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 3, + │ "total": 3 + │ }, + │ "raw": { + │ "input_tokens": 18, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 18, + │ "reasoning_tokens": 0, + │ "tokens": 21 + │ } + ├── ai-sdk-wrapped-model-generate-operation + │ metadata: { + │ "operation": "wrapped-model-generate", + │ "testRunId": "" + │ } + │ ├── generateText [function] + │ │ input: { + │ │ "maxOutputTokens": 24, + │ │ "maxRetries": 2, + │ │ "messages": [ + │ │ { + │ │ "content": "Reply with the single token PARIS and no punctuation.", + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "model": { + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ }, + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "PARIS", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "PARIS", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "responseMessages": "", + │ │ "steps": [ + │ │ { + │ │ "callId": "", + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "PARIS", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "model": { + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "PARIS", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "runtimeContext": {}, + │ │ "stepNumber": 0, + │ │ "toolsContext": {}, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "noCacheTokens": 18 + │ │ }, + │ │ "inputTokens": 18, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 3 + │ │ }, + │ │ "outputTokens": 3, + │ │ "raw": { + │ │ "input_tokens": 18, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 3, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ }, + │ │ "totalTokens": 21 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ ], + │ │ "text": "PARIS", + │ │ "toolCalls": [], + │ │ "toolResults": [], + │ │ "totalUsage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "noCacheTokens": 18 + │ │ }, + │ │ "inputTokens": 18, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 3 + │ │ }, + │ │ "outputTokens": 3, + │ │ "totalTokens": 21 + │ │ }, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "noCacheTokens": 18 + │ │ }, + │ │ "inputTokens": 18, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 3 + │ │ }, + │ │ "outputTokens": 3, + │ │ "totalTokens": 21 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ └── doGenerate [llm] + │ │ input: { + │ │ "maxOutputTokens": 24, + │ │ "messages": [ + │ │ { + │ │ "content": "Reply with the single token PARIS and no punctuation.", + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "model": { + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ }, + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "PARIS", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "response": {}, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "noCacheTokens": 18 + │ │ }, + │ │ "inputTokens": 18, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 3 + │ │ }, + │ │ "outputTokens": 3, + │ │ "raw": { + │ │ "input_tokens": 18, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 3, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ }, + │ │ "totalTokens": 21 + │ │ } + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 3, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 18, + │ │ "tokens": 21 + │ │ } + │ └── doGenerate [llm] + │ input: { + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the single token PARIS and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0, + │ "toolChoice": { + │ "type": "auto" + │ } + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "noCache": 18, + │ "total": 18 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 3, + │ "total": 3 + │ }, + │ "raw": { + │ "input_tokens": 18, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 18, + │ "reasoning_tokens": 0, + │ "tokens": 21 + │ } ├── ai-sdk-generate-operation │ metadata: { │ "operation": "generate", diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-explicit.span-tree.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-explicit.span-tree.json index a495d4aa3..d84c170f0 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-explicit.span-tree.json +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-explicit.span-tree.json @@ -4,6 +4,450 @@ "name": "ai-sdk-instrumentation-root", "type": "task", "children": [ + { + "name": "ai-sdk-direct-model-generate-operation", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with the single token PARIS and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 18, + "total": 18 + }, + "outputTokens": { + "reasoning": 0, + "text": 3, + "total": 3 + }, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 18, + "reasoning_tokens": 0, + "tokens": 21 + } + } + ], + "metadata": { + "operation": "direct-model-generate", + "testRunId": "" + } + }, + { + "name": "ai-sdk-wrapped-model-generate-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": "Reply with the single token PARIS and no punctuation.", + "role": "user" + } + ], + "model": { + "modelId": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "response": {}, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "totalTokens": 21 + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 18, + "tokens": 21 + } + } + ], + "input": { + "maxOutputTokens": 24, + "maxRetries": 2, + "messages": [ + { + "content": "Reply with the single token PARIS and no punctuation.", + "role": "user" + } + ], + "model": { + "modelId": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "responseMessages": "", + "steps": [ + { + "callId": "", + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "model": { + "modelId": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "runtimeContext": {}, + "stepNumber": 0, + "toolsContext": {}, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "totalTokens": 21 + }, + "warnings": [] + } + ], + "text": "PARIS", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "totalTokens": 21 + }, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "totalTokens": 21 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + }, + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with the single token PARIS and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0, + "toolChoice": { + "type": "auto" + } + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 18, + "total": 18 + }, + "outputTokens": { + "reasoning": 0, + "text": 3, + "total": 3 + }, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 18, + "reasoning_tokens": 0, + "tokens": 21 + } + } + ], + "metadata": { + "operation": "wrapped-model-generate", + "testRunId": "" + } + }, { "name": "ai-sdk-generate-operation", "children": [ diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-explicit.span-tree.txt b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-explicit.span-tree.txt index 883a191e7..f9aaeb9e8 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-explicit.span-tree.txt +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-explicit.span-tree.txt @@ -6,6 +6,425 @@ span_tree: "testRunId": "", "workflowVersion": "1.0.8" } + ├── ai-sdk-direct-model-generate-operation + │ metadata: { + │ "operation": "direct-model-generate", + │ "testRunId": "" + │ } + │ └── doGenerate [llm] + │ input: { + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the single token PARIS and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "noCache": 18, + │ "total": 18 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 3, + │ "total": 3 + │ }, + │ "raw": { + │ "input_tokens": 18, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 18, + │ "reasoning_tokens": 0, + │ "tokens": 21 + │ } + ├── ai-sdk-wrapped-model-generate-operation + │ metadata: { + │ "operation": "wrapped-model-generate", + │ "testRunId": "" + │ } + │ ├── generateText [function] + │ │ input: { + │ │ "maxOutputTokens": 24, + │ │ "maxRetries": 2, + │ │ "messages": [ + │ │ { + │ │ "content": "Reply with the single token PARIS and no punctuation.", + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "model": { + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ }, + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "PARIS", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "PARIS", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "responseMessages": "", + │ │ "steps": [ + │ │ { + │ │ "callId": "", + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "PARIS", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "model": { + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "PARIS", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "runtimeContext": {}, + │ │ "stepNumber": 0, + │ │ "toolsContext": {}, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "noCacheTokens": 18 + │ │ }, + │ │ "inputTokens": 18, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 3 + │ │ }, + │ │ "outputTokens": 3, + │ │ "raw": { + │ │ "input_tokens": 18, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 3, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ }, + │ │ "totalTokens": 21 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ ], + │ │ "text": "PARIS", + │ │ "toolCalls": [], + │ │ "toolResults": [], + │ │ "totalUsage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "noCacheTokens": 18 + │ │ }, + │ │ "inputTokens": 18, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 3 + │ │ }, + │ │ "outputTokens": 3, + │ │ "totalTokens": 21 + │ │ }, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "noCacheTokens": 18 + │ │ }, + │ │ "inputTokens": 18, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 3 + │ │ }, + │ │ "outputTokens": 3, + │ │ "totalTokens": 21 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ └── doGenerate [llm] + │ │ input: { + │ │ "maxOutputTokens": 24, + │ │ "messages": [ + │ │ { + │ │ "content": "Reply with the single token PARIS and no punctuation.", + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "model": { + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ }, + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "PARIS", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "response": {}, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "noCacheTokens": 18 + │ │ }, + │ │ "inputTokens": 18, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 3 + │ │ }, + │ │ "outputTokens": 3, + │ │ "raw": { + │ │ "input_tokens": 18, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 3, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ }, + │ │ "totalTokens": 21 + │ │ } + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 3, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 18, + │ │ "tokens": 21 + │ │ } + │ └── doGenerate [llm] + │ input: { + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the single token PARIS and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0, + │ "toolChoice": { + │ "type": "auto" + │ } + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "noCache": 18, + │ "total": 18 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 3, + │ "total": 3 + │ }, + │ "raw": { + │ "input_tokens": 18, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 18, + │ "reasoning_tokens": 0, + │ "tokens": 21 + │ } ├── ai-sdk-generate-operation │ metadata: { │ "operation": "generate", diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-auto-hook.span-tree.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-auto-hook.span-tree.json index cd82f5b3d..8c943bd83 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-auto-hook.span-tree.json @@ -4,6 +4,463 @@ "name": "ai-sdk-instrumentation-root", "type": "task", "children": [ + { + "name": "ai-sdk-direct-model-generate-operation", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with the single token PARIS and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "cacheWrite": 0, + "noCache": 18, + "total": 18 + }, + "outputTokens": { + "reasoning": 0, + "text": 3, + "total": 3 + }, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 18, + "reasoning_tokens": 0, + "tokens": 21 + } + } + ], + "metadata": { + "operation": "direct-model-generate", + "testRunId": "" + } + }, + { + "name": "ai-sdk-wrapped-model-generate-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": "Reply with the single token PARIS and no punctuation.", + "role": "user" + } + ], + "model": { + "modelId": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "response": {}, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "totalTokens": 21 + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_tokens": 3, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 18, + "tokens": 21 + } + } + ], + "input": { + "maxOutputTokens": 24, + "maxRetries": 2, + "messages": [ + { + "content": "Reply with the single token PARIS and no punctuation.", + "role": "user" + } + ], + "model": { + "modelId": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "responseMessages": "", + "steps": [ + { + "callId": "", + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "model": { + "modelId": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "runtimeContext": {}, + "stepNumber": 0, + "toolsContext": {}, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "totalTokens": 21 + }, + "warnings": [] + } + ], + "text": "PARIS", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "totalTokens": 21 + }, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "totalTokens": 21 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + }, + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with the single token PARIS and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0, + "toolChoice": { + "type": "auto" + } + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "cacheWrite": 0, + "noCache": 18, + "total": 18 + }, + "outputTokens": { + "reasoning": 0, + "text": 3, + "total": 3 + }, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 18, + "reasoning_tokens": 0, + "tokens": 21 + } + } + ], + "metadata": { + "operation": "wrapped-model-generate", + "testRunId": "" + } + }, { "name": "ai-sdk-generate-operation", "children": [ diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-auto-hook.span-tree.txt b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-auto-hook.span-tree.txt index e23d0fed6..19d0f6f65 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-auto-hook.span-tree.txt @@ -5,6 +5,438 @@ span_tree: "scenario": "ai-sdk-instrumentation", "testRunId": "" } + ├── ai-sdk-direct-model-generate-operation + │ metadata: { + │ "operation": "direct-model-generate", + │ "testRunId": "" + │ } + │ └── doGenerate [llm] + │ input: { + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the single token PARIS and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "cacheWrite": 0, + │ "noCache": 18, + │ "total": 18 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 3, + │ "total": 3 + │ }, + │ "raw": { + │ "input_tokens": 18, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 18, + │ "reasoning_tokens": 0, + │ "tokens": 21 + │ } + ├── ai-sdk-wrapped-model-generate-operation + │ metadata: { + │ "operation": "wrapped-model-generate", + │ "testRunId": "" + │ } + │ ├── generateText [function] + │ │ input: { + │ │ "maxOutputTokens": 24, + │ │ "maxRetries": 2, + │ │ "messages": [ + │ │ { + │ │ "content": "Reply with the single token PARIS and no punctuation.", + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "model": { + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ }, + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "PARIS", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "PARIS", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "responseMessages": "", + │ │ "steps": [ + │ │ { + │ │ "callId": "", + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "PARIS", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "model": { + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "PARIS", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "runtimeContext": {}, + │ │ "stepNumber": 0, + │ │ "toolsContext": {}, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 18 + │ │ }, + │ │ "inputTokens": 18, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 3 + │ │ }, + │ │ "outputTokens": 3, + │ │ "raw": { + │ │ "input_tokens": 18, + │ │ "input_tokens_details": { + │ │ "cache_write_tokens": 0, + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 3, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ }, + │ │ "totalTokens": 21 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ ], + │ │ "text": "PARIS", + │ │ "toolCalls": [], + │ │ "toolResults": [], + │ │ "totalUsage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 18 + │ │ }, + │ │ "inputTokens": 18, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 3 + │ │ }, + │ │ "outputTokens": 3, + │ │ "totalTokens": 21 + │ │ }, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 18 + │ │ }, + │ │ "inputTokens": 18, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 3 + │ │ }, + │ │ "outputTokens": 3, + │ │ "totalTokens": 21 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ └── doGenerate [llm] + │ │ input: { + │ │ "maxOutputTokens": 24, + │ │ "messages": [ + │ │ { + │ │ "content": "Reply with the single token PARIS and no punctuation.", + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "model": { + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ }, + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "PARIS", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "response": {}, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 18 + │ │ }, + │ │ "inputTokens": 18, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 3 + │ │ }, + │ │ "outputTokens": 3, + │ │ "raw": { + │ │ "input_tokens": 18, + │ │ "input_tokens_details": { + │ │ "cache_write_tokens": 0, + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 3, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ }, + │ │ "totalTokens": 21 + │ │ } + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 3, + │ │ "prompt_cache_creation_tokens": 0, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 18, + │ │ "tokens": 21 + │ │ } + │ └── doGenerate [llm] + │ input: { + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the single token PARIS and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0, + │ "toolChoice": { + │ "type": "auto" + │ } + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "cacheWrite": 0, + │ "noCache": 18, + │ "total": 18 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 3, + │ "total": 3 + │ }, + │ "raw": { + │ "input_tokens": 18, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 18, + │ "reasoning_tokens": 0, + │ "tokens": 21 + │ } ├── ai-sdk-generate-operation │ metadata: { │ "operation": "generate", diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-explicit.span-tree.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-explicit.span-tree.json index cd82f5b3d..8c943bd83 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-explicit.span-tree.json +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-explicit.span-tree.json @@ -4,6 +4,463 @@ "name": "ai-sdk-instrumentation-root", "type": "task", "children": [ + { + "name": "ai-sdk-direct-model-generate-operation", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with the single token PARIS and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "cacheWrite": 0, + "noCache": 18, + "total": 18 + }, + "outputTokens": { + "reasoning": 0, + "text": 3, + "total": 3 + }, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 18, + "reasoning_tokens": 0, + "tokens": 21 + } + } + ], + "metadata": { + "operation": "direct-model-generate", + "testRunId": "" + } + }, + { + "name": "ai-sdk-wrapped-model-generate-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": "Reply with the single token PARIS and no punctuation.", + "role": "user" + } + ], + "model": { + "modelId": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "response": {}, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "totalTokens": 21 + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_tokens": 3, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 18, + "tokens": 21 + } + } + ], + "input": { + "maxOutputTokens": 24, + "maxRetries": 2, + "messages": [ + { + "content": "Reply with the single token PARIS and no punctuation.", + "role": "user" + } + ], + "model": { + "modelId": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "responseMessages": "", + "steps": [ + { + "callId": "", + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "model": { + "modelId": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "runtimeContext": {}, + "stepNumber": 0, + "toolsContext": {}, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "totalTokens": 21 + }, + "warnings": [] + } + ], + "text": "PARIS", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "totalTokens": 21 + }, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "totalTokens": 21 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + }, + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with the single token PARIS and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0, + "toolChoice": { + "type": "auto" + } + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "cacheWrite": 0, + "noCache": 18, + "total": 18 + }, + "outputTokens": { + "reasoning": 0, + "text": 3, + "total": 3 + }, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 18, + "reasoning_tokens": 0, + "tokens": 21 + } + } + ], + "metadata": { + "operation": "wrapped-model-generate", + "testRunId": "" + } + }, { "name": "ai-sdk-generate-operation", "children": [ diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-explicit.span-tree.txt b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-explicit.span-tree.txt index e23d0fed6..19d0f6f65 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-explicit.span-tree.txt +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-explicit.span-tree.txt @@ -5,6 +5,438 @@ span_tree: "scenario": "ai-sdk-instrumentation", "testRunId": "" } + ├── ai-sdk-direct-model-generate-operation + │ metadata: { + │ "operation": "direct-model-generate", + │ "testRunId": "" + │ } + │ └── doGenerate [llm] + │ input: { + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the single token PARIS and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "cacheWrite": 0, + │ "noCache": 18, + │ "total": 18 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 3, + │ "total": 3 + │ }, + │ "raw": { + │ "input_tokens": 18, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 18, + │ "reasoning_tokens": 0, + │ "tokens": 21 + │ } + ├── ai-sdk-wrapped-model-generate-operation + │ metadata: { + │ "operation": "wrapped-model-generate", + │ "testRunId": "" + │ } + │ ├── generateText [function] + │ │ input: { + │ │ "maxOutputTokens": 24, + │ │ "maxRetries": 2, + │ │ "messages": [ + │ │ { + │ │ "content": "Reply with the single token PARIS and no punctuation.", + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "model": { + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ }, + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "PARIS", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "PARIS", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "responseMessages": "", + │ │ "steps": [ + │ │ { + │ │ "callId": "", + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "PARIS", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "model": { + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "PARIS", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "runtimeContext": {}, + │ │ "stepNumber": 0, + │ │ "toolsContext": {}, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 18 + │ │ }, + │ │ "inputTokens": 18, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 3 + │ │ }, + │ │ "outputTokens": 3, + │ │ "raw": { + │ │ "input_tokens": 18, + │ │ "input_tokens_details": { + │ │ "cache_write_tokens": 0, + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 3, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ }, + │ │ "totalTokens": 21 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ ], + │ │ "text": "PARIS", + │ │ "toolCalls": [], + │ │ "toolResults": [], + │ │ "totalUsage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 18 + │ │ }, + │ │ "inputTokens": 18, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 3 + │ │ }, + │ │ "outputTokens": 3, + │ │ "totalTokens": 21 + │ │ }, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 18 + │ │ }, + │ │ "inputTokens": 18, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 3 + │ │ }, + │ │ "outputTokens": 3, + │ │ "totalTokens": 21 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ └── doGenerate [llm] + │ │ input: { + │ │ "maxOutputTokens": 24, + │ │ "messages": [ + │ │ { + │ │ "content": "Reply with the single token PARIS and no punctuation.", + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "model": { + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ }, + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "PARIS", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "response": {}, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 18 + │ │ }, + │ │ "inputTokens": 18, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 3 + │ │ }, + │ │ "outputTokens": 3, + │ │ "raw": { + │ │ "input_tokens": 18, + │ │ "input_tokens_details": { + │ │ "cache_write_tokens": 0, + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 3, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ }, + │ │ "totalTokens": 21 + │ │ } + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 3, + │ │ "prompt_cache_creation_tokens": 0, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 18, + │ │ "tokens": 21 + │ │ } + │ └── doGenerate [llm] + │ input: { + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the single token PARIS and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0, + │ "toolChoice": { + │ "type": "auto" + │ } + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "cacheWrite": 0, + │ "noCache": 18, + │ "total": 18 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 3, + │ "total": 3 + │ }, + │ "raw": { + │ "input_tokens": 18, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 18, + │ "reasoning_tokens": 0, + │ "tokens": 21 + │ } ├── ai-sdk-generate-operation │ metadata: { │ "operation": "generate", diff --git a/e2e/scenarios/ai-sdk-instrumentation/scenario.ai-sdk-v7-explicit.mjs b/e2e/scenarios/ai-sdk-instrumentation/scenario.ai-sdk-v7-explicit.mjs index 5841687a8..067047af3 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/scenario.ai-sdk-v7-explicit.mjs +++ b/e2e/scenarios/ai-sdk-instrumentation/scenario.ai-sdk-v7-explicit.mjs @@ -42,6 +42,7 @@ runMain(async () => { cohere, createCohere, createOpenAI, + directModelWrapping: true, maxTokensKey: "maxOutputTokens", openai, sdkVersion: await getInstalledPackageVersion( diff --git a/e2e/scenarios/ai-sdk-instrumentation/scenario.ai-sdk-v7.mjs b/e2e/scenarios/ai-sdk-instrumentation/scenario.ai-sdk-v7.mjs index f2e7e8539..3cd26c13e 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/scenario.ai-sdk-v7.mjs +++ b/e2e/scenarios/ai-sdk-instrumentation/scenario.ai-sdk-v7.mjs @@ -34,6 +34,7 @@ runAutoAISDKInstrumentationOrExit({ cohere, createCohere, createOpenAI, + directModelWrapping: true, maxTokensKey: "maxOutputTokens", openai, sdkVersion: await getInstalledPackageVersion(import.meta.url, aiPackageName), diff --git a/e2e/scenarios/ai-sdk-instrumentation/scenario.ai-sdk-v7.ts b/e2e/scenarios/ai-sdk-instrumentation/scenario.ai-sdk-v7.ts index 1fad7a0ed..a54f4b686 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/scenario.ai-sdk-v7.ts +++ b/e2e/scenarios/ai-sdk-instrumentation/scenario.ai-sdk-v7.ts @@ -28,6 +28,7 @@ runMain(async () => { cohere, createCohere, createOpenAI, + directModelWrapping: true, maxTokensKey: "maxOutputTokens", openai, sdkVersion: await getInstalledPackageVersion( diff --git a/e2e/scenarios/ai-sdk-instrumentation/scenario.impl.mjs b/e2e/scenarios/ai-sdk-instrumentation/scenario.impl.mjs index f09635def..1824a616e 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/ai-sdk-instrumentation/scenario.impl.mjs @@ -408,17 +408,63 @@ async function runAISDKInstrumentationScenario( const supportsGenerateImage = options.supportsGenerateImage ?? sdkMajorVersion >= 5; const outputObject = createOutputObjectIfSupported(options.ai); - const generateImage = supportsGenerateImage - ? typeof instrumentedAI.generateImage === "function" - ? instrumentedAI.generateImage - : instrumentedAI.experimental_generateImage - : undefined; + let generateImage; + if (supportsGenerateImage) { + generateImage = + typeof instrumentedAI.generateImage === "function" + ? instrumentedAI.generateImage + : instrumentedAI.experimental_generateImage; + } const openaiImageModel = supportsGenerateImage ? openai.image("gpt-image-1-mini") : undefined; + const runDirectModelCalls = async () => { + const model = wrapAISDK(openaiModel); + const callOptions = { + prompt: [ + { + role: "user", + content: [ + { + type: "text", + text: "Reply with the single token PARIS and no punctuation.", + }, + ], + }, + ], + temperature: 0, + maxOutputTokens: 24, + }; + + await runOperation( + "ai-sdk-direct-model-generate-operation", + "direct-model-generate", + async () => { + await model.doGenerate(callOptions); + }, + ); + + await runOperation( + "ai-sdk-wrapped-model-generate-operation", + "wrapped-model-generate", + async () => { + await instrumentedAI.generateText({ + model, + prompt: "Reply with the single token PARIS and no punctuation.", + temperature: 0, + ...tokenLimit(options.maxTokensKey, 24), + }); + }, + ); + }; + await runTracedScenario({ callback: async () => { + if (sdkMajorVersion >= 5 && options.directModelWrapping) { + await runDirectModelCalls(); + } + await runOperation("ai-sdk-generate-operation", "generate", async () => { await instrumentedAI.generateText({ model: openaiModel, @@ -830,9 +876,12 @@ async function runAISDKInstrumentationScenario( } export async function runWrappedAISDKInstrumentation(options) { - await runAISDKInstrumentationScenario(options, { - decorateAI: wrapAISDK, - }); + await runAISDKInstrumentationScenario( + { ...options, directModelWrapping: true }, + { + decorateAI: wrapAISDK, + }, + ); } export async function runAutoAISDKInstrumentation(options) { diff --git a/js/src/instrumentation/plugins/ai-sdk-channels.ts b/js/src/instrumentation/plugins/ai-sdk-channels.ts index ea3f4a880..a26694a90 100644 --- a/js/src/instrumentation/plugins/ai-sdk-channels.ts +++ b/js/src/instrumentation/plugins/ai-sdk-channels.ts @@ -10,6 +10,8 @@ import type { AISDKHarnessAgentCallParams, AISDKHarnessAgentCreateSessionParams, AISDKHarnessAgentSession, + AISDKLanguageModel, + AISDKModelStreamChunk, AISDKRerankParams, AISDKRerankResult, AISDKResult, @@ -27,9 +29,34 @@ type AISDKChannelContext = { span_info?: ChannelSpanInfo; }; +type AISDKModelChannelContext = { + denyOutputPaths?: string[]; + model: AISDKLanguageModel; +}; + +export const BRAINTRUST_WRAPPED_AI_SDK_MODEL = Symbol.for( + "braintrust.ai-sdk.wrapped-model", +); + export const aiSDKChannels = defineChannels( "ai", { + modelGenerate: channel< + [AISDKCallParams], + AISDKResult, + AISDKModelChannelContext + >({ + channelName: "model.doGenerate", + kind: "async", + }), + modelStream: channel< + [AISDKCallParams], + AISDKResult & { stream: ReadableStream }, + AISDKModelChannelContext + >({ + channelName: "model.doStream", + kind: "async", + }), generateText: channel< [AISDKCallParams], AISDKStreamResult, diff --git a/js/src/instrumentation/plugins/ai-sdk-metrics.ts b/js/src/instrumentation/plugins/ai-sdk-metrics.ts new file mode 100644 index 000000000..d4d889e3b --- /dev/null +++ b/js/src/instrumentation/plugins/ai-sdk-metrics.ts @@ -0,0 +1,205 @@ +import { isPromiseLike } from "../../../util"; +import type { AISDKResult } from "../../vendor-sdk-types/ai-sdk"; + +function safeFieldRead(obj: unknown, field: string): unknown { + if (!obj || typeof obj !== "object") { + return undefined; + } + + try { + const value = Reflect.get(obj, field); + if (isPromiseLike(value)) { + void Promise.resolve(value).catch(() => {}); + return undefined; + } + return value; + } catch { + return undefined; + } +} + +function firstNumber(...values: unknown[]): number | undefined { + return values.find((value): value is number => typeof value === "number"); +} + +function parseGatewayCost(cost: unknown): number | undefined { + if (typeof cost === "number") { + return cost; + } + if (typeof cost === "string") { + const parsed = Number.parseFloat(cost); + return Number.isNaN(parsed) ? undefined : parsed; + } + return undefined; +} + +function extractCost(result: AISDKResult): number | undefined { + const steps = safeFieldRead(result, "steps"); + if (Array.isArray(steps)) { + let total = 0; + let found = false; + for (const step of steps) { + const providerMetadata = safeFieldRead(step, "providerMetadata"); + const gateway = safeFieldRead(providerMetadata, "gateway"); + const cost = + parseGatewayCost(safeFieldRead(gateway, "cost")) ?? + parseGatewayCost(safeFieldRead(gateway, "marketCost")); + if (cost !== undefined && cost > 0) { + total += cost; + found = true; + } + } + if (found) { + return total; + } + } + + const providerMetadata = safeFieldRead(result, "providerMetadata"); + const gateway = safeFieldRead(providerMetadata, "gateway"); + const cost = + parseGatewayCost(safeFieldRead(gateway, "cost")) ?? + parseGatewayCost(safeFieldRead(gateway, "marketCost")); + return cost !== undefined && cost > 0 ? cost : undefined; +} + +/** Normalize token and cost metrics across AI SDK provider interface versions. */ +export function extractTokenMetrics( + result: AISDKResult, +): Record { + const totalUsage = safeFieldRead(result, "totalUsage"); + const usageValue = totalUsage ?? safeFieldRead(result, "usage"); + if (!usageValue || typeof usageValue !== "object") { + return {}; + } + + const usage = usageValue; + const inputTokensValue = safeFieldRead(usage, "inputTokens"); + const outputTokensValue = safeFieldRead(usage, "outputTokens"); + const inputTokenDetails = safeFieldRead(usage, "inputTokenDetails"); + const providerMetadata = safeFieldRead(result, "providerMetadata"); + const anthropicMetadata = safeFieldRead(providerMetadata, "anthropic"); + const anthropicUsage = safeFieldRead(anthropicMetadata, "usage"); + const metrics: Record = {}; + + const promptTokens = firstNumber( + safeFieldRead(inputTokensValue, "total"), + inputTokensValue, + safeFieldRead(usage, "promptTokens"), + safeFieldRead(usage, "prompt_tokens"), + ); + const completionTokens = firstNumber( + safeFieldRead(outputTokensValue, "total"), + outputTokensValue, + safeFieldRead(usage, "completionTokens"), + safeFieldRead(usage, "completion_tokens"), + ); + if (completionTokens !== undefined) { + metrics.completion_tokens = completionTokens; + } + + const totalTokens = firstNumber( + safeFieldRead(usage, "totalTokens"), + safeFieldRead(usage, "tokens"), + safeFieldRead(usage, "total_tokens"), + ); + + const promptCachedTokens = firstNumber( + safeFieldRead(inputTokensValue, "cacheRead"), + safeFieldRead(inputTokenDetails, "cacheReadTokens"), + safeFieldRead(usage, "cachedInputTokens"), + safeFieldRead(usage, "promptCachedTokens"), + safeFieldRead(usage, "prompt_cached_tokens"), + safeFieldRead(anthropicUsage, "cache_read_input_tokens"), + ); + if (promptCachedTokens !== undefined) { + metrics.prompt_cached_tokens = promptCachedTokens; + } + + const promptCacheCreationTokens = firstNumber( + safeFieldRead(inputTokensValue, "cacheWrite"), + safeFieldRead(inputTokenDetails, "cacheWriteTokens"), + safeFieldRead(usage, "promptCacheCreationTokens"), + safeFieldRead(usage, "prompt_cache_creation_tokens"), + safeFieldRead(anthropicMetadata, "cacheCreationInputTokens"), + safeFieldRead(anthropicUsage, "cache_creation_input_tokens"), + ); + if (promptCacheCreationTokens !== undefined) { + metrics.prompt_cache_creation_tokens = promptCacheCreationTokens; + } + + const anthropicInputTokens = firstNumber( + safeFieldRead(anthropicUsage, "input_tokens"), + ); + const cacheTokens = + (promptCachedTokens ?? 0) + (promptCacheCreationTokens ?? 0); + const promptTokensExcludeCache = + cacheTokens > 0 && + promptTokens !== undefined && + anthropicInputTokens !== undefined && + promptTokens === anthropicInputTokens; + const normalizedPromptTokens = + promptTokens === undefined + ? undefined + : promptTokens + (promptTokensExcludeCache ? cacheTokens : 0); + if (normalizedPromptTokens !== undefined) { + metrics.prompt_tokens = normalizedPromptTokens; + } + + if (totalTokens !== undefined) { + const totalTokensExcludeCache = + promptTokensExcludeCache && + completionTokens !== undefined && + totalTokens === promptTokens + completionTokens; + metrics.tokens = totalTokens + (totalTokensExcludeCache ? cacheTokens : 0); + } else if ( + normalizedPromptTokens !== undefined && + completionTokens !== undefined + ) { + metrics.tokens = normalizedPromptTokens + completionTokens; + } + + const promptReasoningTokens = firstNumber( + safeFieldRead(usage, "promptReasoningTokens"), + safeFieldRead(usage, "prompt_reasoning_tokens"), + ); + if (promptReasoningTokens !== undefined) { + metrics.prompt_reasoning_tokens = promptReasoningTokens; + } + + const completionCachedTokens = firstNumber( + safeFieldRead(usage, "completionCachedTokens"), + safeFieldRead(usage, "completion_cached_tokens"), + ); + if (completionCachedTokens !== undefined) { + metrics.completion_cached_tokens = completionCachedTokens; + } + + const reasoningTokens = firstNumber( + safeFieldRead(outputTokensValue, "reasoning"), + safeFieldRead(usage, "reasoningTokens"), + safeFieldRead(usage, "completionReasoningTokens"), + safeFieldRead(usage, "completion_reasoning_tokens"), + safeFieldRead(usage, "reasoning_tokens"), + safeFieldRead(usage, "thinkingTokens"), + safeFieldRead(usage, "thinking_tokens"), + ); + if (reasoningTokens !== undefined) { + metrics.completion_reasoning_tokens = reasoningTokens; + metrics.reasoning_tokens = reasoningTokens; + } + + const completionAudioTokens = firstNumber( + safeFieldRead(usage, "completionAudioTokens"), + safeFieldRead(usage, "completion_audio_tokens"), + ); + if (completionAudioTokens !== undefined) { + metrics.completion_audio_tokens = completionAudioTokens; + } + + const cost = extractCost(result); + if (cost !== undefined) { + metrics.estimated_cost = cost; + } + + return metrics; +} diff --git a/js/src/instrumentation/plugins/ai-sdk-plugin.streaming.test.ts b/js/src/instrumentation/plugins/ai-sdk-plugin.streaming.test.ts index 5427d5ca0..170bda408 100644 --- a/js/src/instrumentation/plugins/ai-sdk-plugin.streaming.test.ts +++ b/js/src/instrumentation/plugins/ai-sdk-plugin.streaming.test.ts @@ -16,6 +16,7 @@ import { TestBackgroundLogger, } from "../../logger"; import { wrapAISDK, wrapAgentClass } from "../../wrappers/ai-sdk"; +import { BraintrustMiddleware } from "../../wrappers/ai-sdk/deprecated/BraintrustMiddleware"; import { captureHarnessCreateSessionParent, harnessContinuationParent, @@ -50,6 +51,213 @@ describe("AI SDK streaming instrumentation", () => { _exportsForTestingOnly.clearTestBackgroundLogger(); }); + test("wrapAISDK traces direct model calls with nested usage", async () => { + const model = { + specificationVersion: "v3", + provider: "openai.responses", + modelId: "gpt-4.1-mini", + async doGenerate(this: unknown, _params: any) { + expect(this).toBe(model); + return { + content: [{ type: "text", text: "hello" }], + finishReason: "stop", + usage: { + inputTokens: { total: 12, cacheRead: 5, cacheWrite: 0 }, + outputTokens: { total: 4, reasoning: 1 }, + }, + }; + }, + }; + + const wrapped = wrapAISDK(model); + expect(wrapAISDK(wrapped)).toBe(wrapped); + await wrapped.doGenerate({ prompt: "Say hello" }); + + const spans = (await backgroundLogger.drain()) as any[]; + expect(spans).toHaveLength(1); + expect(spans[0]).toMatchObject({ + span_attributes: { name: "doGenerate", type: "llm" }, + metadata: { + model: "gpt-4.1-mini", + provider: "openai.responses", + }, + metrics: { + prompt_tokens: 12, + completion_tokens: 4, + tokens: 16, + prompt_cached_tokens: 5, + prompt_cache_creation_tokens: 0, + completion_reasoning_tokens: 1, + }, + }); + }); + + test("wrapAISDK traces direct model streams", async () => { + const model = { + specificationVersion: "v2", + provider: "openai.chat", + modelId: "gpt-4o-mini", + async doGenerate(_params: any) { + return {}; + }, + async doStream(_params: any) { + return { + stream: new ReadableStream({ + start(controller) { + controller.enqueue({ type: "text-delta", textDelta: "hello" }); + controller.enqueue({ + type: "finish", + finishReason: "stop", + usage: { inputTokens: 3, outputTokens: 1 }, + }); + controller.close(); + }, + }), + }; + }, + }; + const result = await wrapAISDK(model).doStream!({ prompt: "Say hello" }); + const reader = result.stream.getReader(); + for (;;) { + const next = await reader.read(); + if (next.done) break; + } + + const spans = (await backgroundLogger.drain()) as any[]; + expect(spans).toHaveLength(1); + expect(spans[0]).toMatchObject({ + span_attributes: { name: "doStream", type: "llm" }, + output: { text: "hello", finishReason: "stop" }, + metrics: { + prompt_tokens: 3, + completion_tokens: 1, + tokens: 4, + time_to_first_token: expect.any(Number), + }, + }); + }); + + test("BraintrustMiddleware captures nested AI SDK usage", async () => { + await BraintrustMiddleware({}).wrapGenerate!({ + doGenerate: async () => ({ + content: [{ type: "text", text: "hello" }], + finishReason: "stop", + providerMetadata: { openai: {} }, + usage: { + inputTokens: { total: 8, cacheRead: 3, cacheWrite: 1 }, + outputTokens: { total: 2, reasoning: 1 }, + }, + }), + doStream: async () => ({}), + model: { modelId: "gpt-4.1-mini" }, + params: { prompt: "Say hello" }, + }); + + const spans = (await backgroundLogger.drain()) as any[]; + expect(spans).toHaveLength(1); + expect(spans[0].metrics).toMatchObject({ + prompt_tokens: 8, + completion_tokens: 2, + tokens: 10, + prompt_cached_tokens: 3, + prompt_cache_creation_tokens: 1, + completion_reasoning_tokens: 1, + }); + }); + + test("wrapped models compose with higher-level AI SDK tracing without duplicate LLM spans", async () => { + const model = wrapAISDK({ + specificationVersion: "v3", + provider: "openai.responses", + modelId: "gpt-4.1-mini", + async doGenerate(_params: any) { + return { + text: "hello", + finishReason: "stop", + usage: { + inputTokens: { total: 2 }, + outputTokens: { total: 1 }, + }, + }; + }, + }); + const params = { model, prompt: "Say hello" }; + + await aiSDKChannels.generateText.tracePromise( + () => model.doGenerate(params), + { arguments: [params] } as any, + ); + + const spans = (await backgroundLogger.drain()) as any[]; + expect( + spans.filter((span) => span.span_attributes?.name === "doGenerate"), + ).toHaveLength(1); + expect( + spans.filter((span) => span.span_attributes?.name === "generateText"), + ).toHaveLength(1); + }); + + test("direct model errors propagate and close their span", async () => { + const error = new Error("provider failed"); + const model = wrapAISDK({ + specificationVersion: "v4", + provider: "mock", + modelId: "mock-model", + async doGenerate(_params: any) { + throw error; + }, + }); + + await expect(model.doGenerate({ prompt: "hello" })).rejects.toBe(error); + const spans = (await backgroundLogger.drain()) as any[]; + expect(spans).toHaveLength(1); + expect(spans[0]).toMatchObject({ + span_attributes: { name: "doGenerate", type: "llm" }, + error: expect.stringContaining("provider failed"), + metrics: { end: expect.any(Number) }, + }); + }); + + test("cancelling a direct model stream cancels the provider stream and closes the span", async () => { + let cancelledWith: unknown; + const model = wrapAISDK({ + specificationVersion: "v3", + provider: "mock", + modelId: "mock-model", + async doGenerate(_params: any) { + return {}; + }, + async doStream(_params: any) { + return { + stream: new ReadableStream({ + start(controller) { + controller.enqueue({ type: "text-delta", textDelta: "hello" }); + }, + cancel(reason) { + cancelledWith = reason; + }, + }), + }; + }, + }); + + const result = await model.doStream!({ prompt: "hello" }); + const reader = result.stream.getReader(); + await reader.read(); + await reader.cancel("user stopped"); + + expect(cancelledWith).toBe("user stopped"); + const spans = (await backgroundLogger.drain()) as any[]; + expect(spans).toHaveLength(1); + expect(spans[0]).toMatchObject({ + span_attributes: { name: "doStream", type: "llm" }, + metadata: { + usage_unavailable_reason: "ai_sdk_stream_cancelled_without_usage", + }, + metrics: { end: expect.any(Number) }, + }); + }); + describe.each(["promise", "generator"])("%s tool execution", (kind) => { test.each([ { options: { toolCallId: "tool-1" }, expectedId: "tool-1" }, diff --git a/js/src/instrumentation/plugins/ai-sdk-plugin.test.ts b/js/src/instrumentation/plugins/ai-sdk-plugin.test.ts index 231f52d22..78da92712 100644 --- a/js/src/instrumentation/plugins/ai-sdk-plugin.test.ts +++ b/js/src/instrumentation/plugins/ai-sdk-plugin.test.ts @@ -31,6 +31,7 @@ import { processAISDKWorkflowAgentModelCallInput, processAISDKOutput as processAISDKOutputActual, processAISDKGenerateImageOutput, + extractTokenMetrics, } from "./ai-sdk-plugin"; import iso from "../../isomorph"; import { serializeAISDKToolsForLogging } from "../../wrappers/ai-sdk/tool-serialization"; @@ -1072,6 +1073,98 @@ describe("AI SDK utility functions", () => { }); }); + it("should extract nested cache and reasoning usage", () => { + const metrics = extractTokenMetrics({ + usage: { + inputTokens: { + total: 100, + cacheRead: 40, + cacheWrite: 10, + }, + outputTokens: { total: 20, reasoning: 7 }, + }, + }); + + expect(metrics).toEqual({ + prompt_tokens: 100, + completion_tokens: 20, + tokens: 120, + prompt_cached_tokens: 40, + prompt_cache_creation_tokens: 10, + completion_reasoning_tokens: 7, + reasoning_tokens: 7, + }); + }); + + it("should preserve explicit zero values in nested usage", () => { + expect( + extractTokenMetrics({ + usage: { + inputTokens: { total: 0, cacheRead: 0, cacheWrite: 0 }, + outputTokens: { total: 0, reasoning: 0 }, + }, + }), + ).toMatchObject({ + prompt_tokens: 0, + completion_tokens: 0, + tokens: 0, + prompt_cached_tokens: 0, + prompt_cache_creation_tokens: 0, + completion_reasoning_tokens: 0, + }); + }); + + it("should not add Anthropic cache tokens to nested inclusive totals", () => { + expect( + extractTokenMetrics({ + usage: { + inputTokens: { total: 100, cacheRead: 40, cacheWrite: 10 }, + outputTokens: { total: 20 }, + }, + providerMetadata: { + anthropic: { + usage: { + cache_read_input_tokens: 40, + cache_creation_input_tokens: 10, + }, + }, + }, + }), + ).toMatchObject({ + prompt_tokens: 100, + completion_tokens: 20, + tokens: 120, + }); + }); + + it("should add Anthropic cache tokens to explicitly exclusive totals", () => { + expect( + extractTokenMetrics({ + usage: { + inputTokens: 3, + outputTokens: 7, + totalTokens: 10, + }, + providerMetadata: { + anthropic: { + usage: { + input_tokens: 3, + output_tokens: 7, + cache_read_input_tokens: 4516, + cache_creation_input_tokens: 0, + }, + }, + }, + }), + ).toMatchObject({ + prompt_tokens: 4519, + completion_tokens: 7, + tokens: 4526, + prompt_cached_tokens: 4516, + prompt_cache_creation_tokens: 0, + }); + }); + it("should handle flat inputTokens/outputTokens", () => { const result = { usage: { @@ -1264,7 +1357,7 @@ describe("AI SDK utility functions", () => { }); it("should return empty metrics for null result", () => { - const metrics = extractTokenMetrics(null); + const metrics = extractTokenMetrics(null as any); expect(metrics).toEqual({}); }); @@ -1971,100 +2064,6 @@ function extractGetterValues(obj: any): any { return getterValues; } -function extractTokenMetrics(result: any): Record { - const metrics: Record = {}; - - let usage = result?.totalUsage || result?.usage; - - if (!usage && result) { - try { - if ("totalUsage" in result && typeof result.totalUsage !== "function") { - usage = result.totalUsage; - } else if ("usage" in result && typeof result.usage !== "function") { - usage = result.usage; - } - } catch { - // Ignore errors accessing getters - } - } - - if (!usage) { - return metrics; - } - - const promptTokens = firstNumber( - usage.inputTokens?.total, - usage.inputTokens, - usage.promptTokens, - usage.prompt_tokens, - ); - if (promptTokens !== undefined) { - metrics.prompt_tokens = promptTokens; - } - - const completionTokens = firstNumber( - usage.outputTokens?.total, - usage.outputTokens, - usage.completionTokens, - usage.completion_tokens, - ); - if (completionTokens !== undefined) { - metrics.completion_tokens = completionTokens; - } - - const totalTokens = firstNumber( - usage.totalTokens, - usage.tokens, - usage.total_tokens, - ); - if (totalTokens !== undefined) { - metrics.tokens = totalTokens; - } else if (promptTokens !== undefined && completionTokens !== undefined) { - metrics.tokens = promptTokens + completionTokens; - } - - const promptCachedTokens = firstNumber( - usage.inputTokens?.cacheRead, - usage.inputTokenDetails?.cacheReadTokens, - usage.cachedInputTokens, - usage.promptCachedTokens, - usage.prompt_cached_tokens, - ); - if (promptCachedTokens !== undefined) { - metrics.prompt_cached_tokens = promptCachedTokens; - } - - const promptCacheCreationTokens = firstNumber( - usage.inputTokens?.cacheWrite, - usage.inputTokenDetails?.cacheWriteTokens, - usage.promptCacheCreationTokens, - usage.prompt_cache_creation_tokens, - extractAnthropicCacheCreationTokens(result), - ); - if (promptCacheCreationTokens !== undefined) { - metrics.prompt_cache_creation_tokens = promptCacheCreationTokens; - } - - const cost = extractCostFromResult(result); - if (cost !== undefined) { - metrics.estimated_cost = cost; - } - - return metrics; -} - -function extractAnthropicCacheCreationTokens(result: any): number | undefined { - const anthropicMetadata = result?.providerMetadata?.anthropic; - if (!anthropicMetadata || typeof anthropicMetadata !== "object") { - return undefined; - } - - return firstNumber( - anthropicMetadata.cacheCreationInputTokens, - anthropicMetadata.usage?.cache_creation_input_tokens, - ); -} - function extractCostFromResult(result: any): number | undefined { if (result?.steps && Array.isArray(result.steps) && result.steps.length > 0) { let totalCost = 0; diff --git a/js/src/instrumentation/plugins/ai-sdk-plugin.ts b/js/src/instrumentation/plugins/ai-sdk-plugin.ts index bfe41694c..578dd773a 100644 --- a/js/src/instrumentation/plugins/ai-sdk-plugin.ts +++ b/js/src/instrumentation/plugins/ai-sdk-plugin.ts @@ -48,7 +48,12 @@ import { unregisterWorkflowAgentWrapperSpan, } from "../../wrappers/ai-sdk/workflow-agent-context"; import { zodToJsonSchema } from "../../zod/utils"; -import { aiSDKChannels, harnessAgentChannels } from "./ai-sdk-channels"; +import { + aiSDKChannels, + BRAINTRUST_WRAPPED_AI_SDK_MODEL, + harnessAgentChannels, +} from "./ai-sdk-channels"; +import { extractTokenMetrics } from "./ai-sdk-metrics"; import { currentCloudflareThinkSpan } from "./cloudflare-think-context"; import type { AISDK, @@ -195,8 +200,12 @@ export class AISDKPlugin extends BasePlugin { const denyOutputPaths = this.config.denyOutputPaths || DEFAULT_DENY_OUTPUT_PATHS; - this.unsubscribers.push(interceptAISDKV7TelemetryDispatcher()); - this.unsubscribers.push(subscribeToHarnessAgentCreateSession()); + this.unsubscribers.push( + interceptAISDKV7TelemetryDispatcher(), + interceptAISDKModelGenerate(denyOutputPaths), + interceptAISDKModelStream(denyOutputPaths), + subscribeToHarnessAgentCreateSession(), + ); this.unsubscribers.push( subscribeToHarnessContinuation( harnessAgentChannels.continueGenerate, @@ -882,6 +891,191 @@ function interceptAISDKV7TelemetryDispatcher(): () => void { ); } +function buildAISDKModelSpanArgs( + name: "doGenerate" | "doStream", + params: AISDKCallParams, + model: AISDKLanguageModel, +) { + return withSpanInstrumentationName( + { + name, + spanAttributes: { type: SpanTypeAttribute.LLM }, + event: buildAISDKModelStartEvent( + params, + buildAISDKChildMetadata(model), + {}, + ), + }, + INSTRUMENTATION_NAMES.AI_SDK, + ); +} + +function interceptAISDKModelGenerate( + defaultDenyOutputPaths: string[], +): () => void { + return aiSDKChannels.modelGenerate.intercept( + async (target, thisArg, [params], additional) => { + const span = _internalStartSpanWithInitialMerge( + buildAISDKModelSpanArgs("doGenerate", params, additional.model), + ); + try { + const result = await withCurrent(span, () => + Reflect.apply(target, thisArg, [params]), + ); + const metrics = extractTokenMetrics(result); + span.log({ + output: processAISDKOutput( + result, + additional.denyOutputPaths ?? defaultDenyOutputPaths, + ), + metrics, + ...mergeMetadataPayload( + buildResolvedMetadataPayload(result), + buildMissingUsageMetadata( + result, + metrics, + "ai_sdk_result_missing_usage", + ), + ), + }); + return result; + } catch (error) { + span.log({ error: toLoggedError(error) }); + throw error; + } finally { + span.end(); + } + }, + ); +} + +function interceptAISDKModelStream( + defaultDenyOutputPaths: string[], +): () => void { + return aiSDKChannels.modelStream.intercept( + async (target, thisArg, [params], additional) => { + const span = _internalStartSpanWithInitialMerge( + buildAISDKModelSpanArgs("doStream", params, additional.model), + ); + const startTime = getCurrentUnixTimestamp(); + let result: AISDKResult & { + stream: ReadableStream; + }; + try { + result = await withCurrent(span, () => + Reflect.apply(target, thisArg, [params]), + ); + } catch (error) { + span.log({ error: toLoggedError(error) }); + span.end(); + throw error; + } + + const reader = result.stream.getReader(); + const output: Record = {}; + const textParts: string[] = []; + const reasoningParts: string[] = []; + const toolCalls: unknown[] = []; + let firstChunkTime: number | undefined; + let ended = false; + + const logAndEnd = (usageUnavailableReason?: string) => { + if (ended) { + return; + } + ended = true; + output.text = textParts.join(""); + output.reasoning = reasoningParts.join(""); + output.toolCalls = toolCalls; + const aggregatedResult = output as AISDKResult; + const metrics = extractTokenMetrics(aggregatedResult); + if (firstChunkTime !== undefined) { + metrics.time_to_first_token = Math.max( + firstChunkTime - startTime, + 1e-6, + ); + } + const missingUsageMetadata = usageUnavailableReason + ? { usage_unavailable_reason: usageUnavailableReason } + : buildMissingUsageMetadata( + aggregatedResult, + metrics, + "ai_sdk_result_missing_usage", + ); + span.log({ + output: processAISDKOutput( + aggregatedResult, + additional.denyOutputPaths ?? defaultDenyOutputPaths, + ), + metrics, + ...mergeMetadataPayload( + buildResolvedMetadataPayload(aggregatedResult), + missingUsageMetadata, + ), + }); + span.end(); + }; + + const processChunk = (chunk: AISDKModelStreamChunk) => { + if (firstChunkTime === undefined && isAISDKContentStreamChunk(chunk)) { + firstChunkTime = getCurrentUnixTimestamp(); + } + switch (chunk.type) { + case "text-delta": + textParts.push(extractTextDelta(chunk)); + break; + case "reasoning-delta": + reasoningParts.push(chunk.delta ?? chunk.text ?? ""); + break; + case "tool-call": + toolCalls.push(chunk); + break; + case "object": + output.object = chunk.object; + break; + case "finish": + output.finishReason = chunk.finishReason; + output.usage = chunk.usage; + if (chunk.providerMetadata !== undefined) { + output.providerMetadata = chunk.providerMetadata; + } + logAndEnd(); + break; + } + }; + + return { + ...result, + stream: new ReadableStream({ + async pull(controller) { + try { + const next = await withCurrent(span, () => reader.read()); + if (next.done) { + logAndEnd("ai_sdk_stream_finished_without_usage"); + controller.close(); + return; + } + processChunk(next.value); + controller.enqueue(next.value); + } catch (error) { + span.log({ error: toLoggedError(error) }); + logAndEnd("ai_sdk_stream_errored_without_usage"); + controller.error(error); + } + }, + async cancel(reason) { + try { + await reader.cancel(reason); + } finally { + logAndEnd("ai_sdk_stream_cancelled_without_usage"); + } + }, + }), + }; + }, + ); +} + function patchAISDKV7TelemetryDispatcher( dispatcher: unknown, telemetry: AISDKV7Telemetry, @@ -2233,6 +2427,15 @@ function prepareAISDKChildTracing( return resolvedModel; } + if ( + (resolvedModel as Record)[ + BRAINTRUST_WRAPPED_AI_SDK_MODEL + ] + ) { + modelWrapped = true; + return resolvedModel; + } + const existingWrappedModel = patchedModels.get(resolvedModel); if (existingWrappedModel) { modelWrapped = true; @@ -3799,159 +4002,7 @@ export function processAISDKRerankOutput( return undefined; } -/** - * Extract token metrics from AI SDK result. - */ -export function extractTokenMetrics( - result: AISDKResult, -): Record { - const metrics: Record = {}; - - let usage: AISDKUsage | undefined; - const totalUsageValue = safeResultFieldRead(result, "totalUsage"); - if (totalUsageValue !== undefined && !isPromiseLike(totalUsageValue)) { - usage = totalUsageValue as AISDKUsage; - } - - if (!usage) { - const usageValue = safeResultFieldRead(result, "usage"); - if (usageValue !== undefined && !isPromiseLike(usageValue)) { - usage = usageValue as AISDKUsage; - } - } - - if (!usage) { - return metrics; - } - - // Extract token counts - const promptTokens = firstNumber( - usage.inputTokens?.total, - usage.inputTokens, - usage.promptTokens, - usage.prompt_tokens, - ); - if (promptTokens !== undefined) { - metrics.prompt_tokens = promptTokens; - } - - const completionTokens = firstNumber( - usage.outputTokens?.total, - usage.outputTokens, - usage.completionTokens, - usage.completion_tokens, - ); - if (completionTokens !== undefined) { - metrics.completion_tokens = completionTokens; - } - - const totalTokens = firstNumber( - usage.totalTokens, - usage.tokens, - usage.total_tokens, - ); - if (totalTokens !== undefined) { - metrics.tokens = totalTokens; - } else if (promptTokens !== undefined && completionTokens !== undefined) { - metrics.tokens = promptTokens + completionTokens; - } - - const promptCachedTokens = firstNumber( - usage.inputTokens?.cacheRead, - usage.inputTokenDetails?.cacheReadTokens, - usage.cachedInputTokens, - usage.promptCachedTokens, - usage.prompt_cached_tokens, - ); - if (promptCachedTokens !== undefined) { - metrics.prompt_cached_tokens = promptCachedTokens; - } - - const promptCacheCreationTokens = firstNumber( - usage.inputTokens?.cacheWrite, - usage.inputTokenDetails?.cacheWriteTokens, - usage.promptCacheCreationTokens, - usage.prompt_cache_creation_tokens, - extractAnthropicCacheCreationTokens(result), - ); - if (promptCacheCreationTokens !== undefined) { - metrics.prompt_cache_creation_tokens = promptCacheCreationTokens; - } - - const promptReasoningTokens = firstNumber( - usage.promptReasoningTokens, - usage.prompt_reasoning_tokens, - ); - if (promptReasoningTokens !== undefined) { - metrics.prompt_reasoning_tokens = promptReasoningTokens; - } - - const completionCachedTokens = firstNumber( - usage.completionCachedTokens, - usage.completion_cached_tokens, - ); - if (completionCachedTokens !== undefined) { - metrics.completion_cached_tokens = completionCachedTokens; - } - - const reasoningTokenCount = firstNumber( - usage.outputTokens?.reasoning, - usage.reasoningTokens, - usage.completionReasoningTokens, - usage.completion_reasoning_tokens, - usage.reasoning_tokens, - usage.thinkingTokens, - usage.thinking_tokens, - ); - if (reasoningTokenCount !== undefined) { - metrics.completion_reasoning_tokens = reasoningTokenCount; - metrics.reasoning_tokens = reasoningTokenCount; - } - - const completionAudioTokens = firstNumber( - usage.completionAudioTokens, - usage.completion_audio_tokens, - ); - if (completionAudioTokens !== undefined) { - metrics.completion_audio_tokens = completionAudioTokens; - } - - // Extract cost from gateway routing if available - const cost = extractCostFromResult(result); - if (cost !== undefined) { - metrics.estimated_cost = cost; - } - - return metrics; -} - -function extractAnthropicCacheCreationTokens( - result: AISDKResult, -): number | undefined { - const providerMetadata = safeSerializableFieldRead( - result, - "providerMetadata", - ) as Record | undefined; - const anthropicMetadata = providerMetadata?.anthropic as - | Record - | undefined; - if (!anthropicMetadata) { - return undefined; - } - - return firstNumber( - anthropicMetadata.cacheCreationInputTokens, - (anthropicMetadata.usage as Record | undefined) - ?.cache_creation_input_tokens, - ); -} - -function safeResultFieldRead( - result: AISDKResult, - field: "usage" | "totalUsage", -): unknown { - return safeSerializableFieldRead(result, field); -} +export { extractTokenMetrics }; function safeSerializableFieldRead( obj: Record | AISDKResult, @@ -4219,76 +4270,6 @@ function extractGatewayRoutingInfo(result: AISDKResult): { return null; } -/** - * Extract cost from result's providerMetadata. - */ -function extractCostFromResult(result: AISDKResult): number | undefined { - // Check for cost in steps (multi-step results) - const steps = safeSerializableFieldRead(result, "steps"); - if (Array.isArray(steps) && steps.length > 0) { - let totalCost = 0; - let foundCost = false; - for (const step of steps) { - const gateway = step?.providerMetadata?.gateway; - const stepCost = - parseGatewayCost(gateway?.cost) || - parseGatewayCost(gateway?.marketCost); - if (stepCost !== undefined && stepCost > 0) { - totalCost += stepCost; - foundCost = true; - } - } - if (foundCost) { - return totalCost; - } - } - - // Check direct providerMetadata - const providerMetadata = safeSerializableFieldRead( - result, - "providerMetadata", - ); - const gateway = (providerMetadata as { gateway?: any } | undefined)?.gateway; - const directCost = - parseGatewayCost(gateway?.cost) || parseGatewayCost(gateway?.marketCost); - if (directCost !== undefined && directCost > 0) { - return directCost; - } - - return undefined; -} - -/** - * Parse gateway cost value. - */ -function parseGatewayCost(cost: unknown): number | undefined { - if (cost === undefined || cost === null) { - return undefined; - } - if (typeof cost === "number") { - return cost; - } - if (typeof cost === "string") { - const parsed = parseFloat(cost); - if (!isNaN(parsed)) { - return parsed; - } - } - return undefined; -} - -/** - * Get first number from a list of values. - */ -function firstNumber(...values: unknown[]): number | undefined { - for (const v of values) { - if (typeof v === "number") { - return v; - } - } - return undefined; -} - /** * Deep copy an object via JSON serialization. */ diff --git a/js/src/vendor-sdk-types/ai-sdk-common.ts b/js/src/vendor-sdk-types/ai-sdk-common.ts index 5c8d406ba..c487710a4 100644 --- a/js/src/vendor-sdk-types/ai-sdk-common.ts +++ b/js/src/vendor-sdk-types/ai-sdk-common.ts @@ -12,7 +12,7 @@ interface AISDKTokenBucket { [key: string]: unknown; } -type AISDKTokenCount = number & AISDKTokenBucket; +type AISDKTokenCount = number | AISDKTokenBucket; export interface AISDKUsage { inputTokens?: AISDKTokenCount; @@ -92,6 +92,7 @@ export interface AISDKGenerateImageParams extends Omit< } export interface AISDKLanguageModel { + specificationVersion?: string; modelId?: string; provider?: string; supportsStructuredOutputs?: boolean; @@ -101,6 +102,7 @@ export interface AISDKLanguageModel { ) => Promise }>; _braintrustWrapped?: boolean; [key: string]: unknown; + [key: symbol]: unknown; } export type AISDKModel = string | AISDKLanguageModel; diff --git a/js/src/wrappers/ai-sdk/README.md b/js/src/wrappers/ai-sdk/README.md index c3080610e..a4ef9d080 100644 --- a/js/src/wrappers/ai-sdk/README.md +++ b/js/src/wrappers/ai-sdk/README.md @@ -2,6 +2,21 @@ This wrapper provides Braintrust logging integration for the Vercel AI SDK. +`wrapAISDK` accepts either the AI SDK namespace or an AI SDK 5–7 language model. +Wrapping a model instruments direct `doGenerate()` and `doStream()` calls while preserving the model's type and method receiver. + +```typescript +import { wrapAISDK } from "braintrust"; +import { openai } from "@ai-sdk/openai"; + +const model = wrapAISDK(openai("gpt-4.1-mini")); +const result = await model.doGenerate({ + prompt: [{ role: "user", content: [{ type: "text", text: "Say hello" }] }], +}); +``` + +`BraintrustMiddleware` remains available for compatibility, but new direct-model integrations should use `wrapAISDK(model)`. + ## Test Structure Tests are organized to support multiple AI SDK versions: diff --git a/js/src/wrappers/ai-sdk/ai-sdk.ts b/js/src/wrappers/ai-sdk/ai-sdk.ts index 1173a0e0d..aab0f4ea3 100644 --- a/js/src/wrappers/ai-sdk/ai-sdk.ts +++ b/js/src/wrappers/ai-sdk/ai-sdk.ts @@ -3,6 +3,7 @@ import { SpanTypeAttribute } from "../../../util"; import { aiSDKChannels, + BRAINTRUST_WRAPPED_AI_SDK_MODEL, harnessAgentChannels, } from "../../instrumentation/plugins/ai-sdk-channels"; import type { @@ -20,8 +21,10 @@ import type { AISDKHarnessAgentGenerateFunction, AISDKHarnessAgentInstance, AISDKHarnessAgentStreamFunction, + AISDKLanguageModel, AISDKRerankFunction, AISDKRerankParams, + AISDKResult, AISDKStreamFunction, AISDKWorkflowAgentClass, } from "../../vendor-sdk-types/ai-sdk"; @@ -40,6 +43,9 @@ type SpanInfo = { }; type AISDKNamespaceObject = Record; +type SupportedAISDKLanguageModel = AISDKLanguageModel & { + doGenerate: NonNullable; +}; /** * Detects if an object is an ES module namespace (ModuleRecord). @@ -110,6 +116,10 @@ export function wrapAISDK(aiSDK: T, options: WrapAISDKOptions = {}): T { return aiSDK; } + if (isSupportedLanguageModel(aiSDK)) { + return wrapLanguageModel(aiSDK, options) as T; + } + const typedAISDK = aiSDK as unknown as AISDK; // Handle ES module namespaces (ModuleRecords) that have non-configurable properties. @@ -166,6 +176,60 @@ export function wrapAISDK(aiSDK: T, options: WrapAISDKOptions = {}): T { }) as T; } +function isSupportedLanguageModel( + value: unknown, +): value is SupportedAISDKLanguageModel { + if (!value || typeof value !== "object") { + return false; + } + const model = value as AISDKLanguageModel; + return ( + ["v2", "v3", "v4"].includes(model.specificationVersion ?? "") && + typeof model.doGenerate === "function" + ); +} + +function wrapLanguageModel( + model: SupportedAISDKLanguageModel, + options: WrapAISDKOptions, +): AISDKLanguageModel { + if (model[BRAINTRUST_WRAPPED_AI_SDK_MODEL]) { + return model; + } + + const wrapped = Object.create( + Object.getPrototypeOf(model), + Object.getOwnPropertyDescriptors(model), + ) as AISDKLanguageModel; + Object.defineProperty(wrapped, BRAINTRUST_WRAPPED_AI_SDK_MODEL, { + configurable: false, + enumerable: false, + value: true, + }); + + const doGenerate = model.doGenerate; + wrapped.doGenerate = (params: AISDKCallParams) => + aiSDKChannels.modelGenerate.invoke(doGenerate, model, [params], { + model, + ...(options.denyOutputPaths + ? { denyOutputPaths: options.denyOutputPaths } + : {}), + }) as Promise; + + if (typeof model.doStream === "function") { + const doStream = model.doStream; + wrapped.doStream = (params: AISDKCallParams) => + aiSDKChannels.modelStream.invoke(doStream, model, [params], { + model, + ...(options.denyOutputPaths + ? { denyOutputPaths: options.denyOutputPaths } + : {}), + }) as ReturnType>; + } + + return wrapped; +} + function isHarnessAgentInstance( instance: AISDKAgentInstance, ): instance is AISDKAgentInstance & AISDKHarnessAgentInstance { diff --git a/js/src/wrappers/ai-sdk/deprecated/BraintrustMiddleware.ts b/js/src/wrappers/ai-sdk/deprecated/BraintrustMiddleware.ts index d483ed02b..30232cc3e 100644 --- a/js/src/wrappers/ai-sdk/deprecated/BraintrustMiddleware.ts +++ b/js/src/wrappers/ai-sdk/deprecated/BraintrustMiddleware.ts @@ -6,12 +6,12 @@ import { INSTRUMENTATION_NAMES, withSpanInstrumentationName, } from "../../../span-origin"; -import { - extractAnthropicCacheTokens, - finalizeAnthropicTokens, - toNumericMetrics, -} from "../../anthropic-tokens-util"; import { processInputAttachments } from "../../attachment-utils"; +import { extractTokenMetrics } from "../../../instrumentation/plugins/ai-sdk-metrics"; +import type { + AISDKProviderMetadata, + AISDKUsage, +} from "../../../vendor-sdk-types/ai-sdk-common"; function detectProviderFromResult(result: { providerMetadata?: Record; @@ -39,16 +39,6 @@ function extractModelFromResult(result: { return undefined; } -function extractModelFromWrapGenerateCallback(model: { - modelId?: string; - config?: Record; - specificationVersion?: string; - provider?: string; - supportedUrls?: Record; -}): string | undefined { - return model?.modelId; -} - function camelToSnake(str: string): string { return str.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`); } @@ -69,76 +59,6 @@ function extractModelParameters( return modelParams; } -function getNumberProperty(obj: unknown, key: string): number | undefined { - if (!obj || typeof obj !== "object" || !(key in obj)) { - return undefined; - } - const value = Reflect.get(obj, key); - return typeof value === "number" ? value : undefined; -} - -function normalizeUsageMetrics( - usage: unknown, - provider?: string, - providerMetadata?: Record, -): Record { - const metrics: Record = {}; - - // Standard AI SDK usage fields - const inputTokens = getNumberProperty(usage, "inputTokens"); - if (inputTokens !== undefined) { - metrics.prompt_tokens = inputTokens; - } - - const outputTokens = getNumberProperty(usage, "outputTokens"); - if (outputTokens !== undefined) { - metrics.completion_tokens = outputTokens; - } - - const totalTokens = getNumberProperty(usage, "totalTokens"); - if (totalTokens !== undefined) { - metrics.tokens = totalTokens; - } - - const reasoningTokens = getNumberProperty(usage, "reasoningTokens"); - if (reasoningTokens !== undefined) { - metrics.completion_reasoning_tokens = reasoningTokens; - } - - const cachedInputTokens = getNumberProperty(usage, "cachedInputTokens"); - if (cachedInputTokens !== undefined) { - metrics.prompt_cached_tokens = cachedInputTokens; - } - - // Anthropic-specific cache token handling - if (provider === "anthropic") { - const anthropicMetadata = providerMetadata?.anthropic as any; - - if (anthropicMetadata) { - const cacheReadTokens = - getNumberProperty(anthropicMetadata.usage, "cache_read_input_tokens") || - 0; - const cacheCreationTokens = - getNumberProperty( - anthropicMetadata.usage, - "cache_creation_input_tokens", - ) || 0; - - const cacheTokens = extractAnthropicCacheTokens( - cacheReadTokens, - cacheCreationTokens, - ); - Object.assign(metrics, cacheTokens); - - // Use the returned object: finalization can drop cache-creation metrics, - // and merging it back over `metrics` would keep them. - return toNumericMetrics(finalizeAnthropicTokens(metrics)); - } - } - - return metrics; -} - function normalizeFinishReason(reason: any): string | undefined { if (typeof reason !== "string") return undefined; return reason.replace(/-/g, "_"); @@ -309,10 +229,7 @@ export function BraintrustMiddleware( if (model !== undefined) { metadata.model = model; } else if (modelFromWrapGenerate) { - // Use the model from the wrapGenerate call if it's not in the result - const modelId = extractModelFromWrapGenerateCallback( - modelFromWrapGenerate, - ); + const modelId = modelFromWrapGenerate.modelId; if (modelId) { metadata.model = modelId; } @@ -329,11 +246,10 @@ export function BraintrustMiddleware( ? buildAssistantOutputWithToolCalls(result, toolCalls) : (result as any)?.content, metadata, - metrics: normalizeUsageMetrics( - result.usage, - provider, - result.providerMetadata, - ), + metrics: extractTokenMetrics({ + usage: result.usage, + providerMetadata: result.providerMetadata, + }), }); return result; @@ -375,9 +291,9 @@ export function BraintrustMiddleware( const textChunks: string[] = []; const toolBlocks: any[] = []; - let finalUsage: unknown = {}; + let finalUsage: AISDKUsage = {}; let finalFinishReason: unknown = undefined; - let providerMetadata: Record = {}; + let providerMetadata: AISDKProviderMetadata = {}; const transformStream = new TransformStream({ transform(chunk: any, controller: any) { @@ -458,11 +374,10 @@ export function BraintrustMiddleware( span.log({ output, metadata, - metrics: normalizeUsageMetrics( - finalUsage, - provider, + metrics: extractTokenMetrics({ + usage: finalUsage, providerMetadata, - ), + }), }); span.end(); From ae7e2bf671d5596500b6a41698c3aeeed13a7848 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Thu, 17 Sep 2026 14:54:31 +0000 Subject: [PATCH 2/2] Update PR #2497 --- .changeset/fix-ai-sdk-middleware-metrics.md | 5 + e2e/helpers/mock-braintrust-server.test.ts | 146 +++++++++++++----- e2e/helpers/mock-braintrust-server.ts | 22 ++- .../flue-v2-latest.span-tree.json | 3 +- .../flue-v2-latest.span-tree.txt | 3 +- .../__snapshots__/flue-v2.span-tree.json | 3 +- .../__snapshots__/flue-v2.span-tree.txt | 3 +- .../flue-instrumentation/scenario.v2.ts | 7 +- .../flue-instrumentation/v2-assertions.ts | 6 + 9 files changed, 151 insertions(+), 47 deletions(-) create mode 100644 .changeset/fix-ai-sdk-middleware-metrics.md diff --git a/.changeset/fix-ai-sdk-middleware-metrics.md b/.changeset/fix-ai-sdk-middleware-metrics.md new file mode 100644 index 000000000..b2ae44032 --- /dev/null +++ b/.changeset/fix-ai-sdk-middleware-metrics.md @@ -0,0 +1,5 @@ +--- +"braintrust": minor +--- + +feat: Fix token metrics with AI SDK middleware diff --git a/e2e/helpers/mock-braintrust-server.test.ts b/e2e/helpers/mock-braintrust-server.test.ts index 7bdd1f25d..4acf647ba 100644 --- a/e2e/helpers/mock-braintrust-server.test.ts +++ b/e2e/helpers/mock-braintrust-server.test.ts @@ -90,45 +90,111 @@ describe("production forwarding", () => { }, ); - it("reports a failed write without preventing later queued writes", async () => { - const received: number[] = []; - const upstream = createServer(async (req, res) => { - let body = ""; - for await (const chunk of req) body += chunk; - const { sequence } = JSON.parse(body); - received.push(sequence); - res.statusCode = sequence === 1 ? 500 : 200; - res.end(sequence === 1 ? "initial write failed" : "{}"); - }); - await new Promise((resolve) => - upstream.listen(0, "127.0.0.1", resolve), - ); - const url = `http://127.0.0.1:${(upstream.address() as AddressInfo).port}`; - const server = await startMockBraintrustServer({ - prodForwarding: { - apiKey: "test-only-key", - apiUrl: url, - appUrl: url, - orgId: "org", - orgName: "org", - projectId: "project", - projectName: "tmp-luca-forwarding-test", - }, - }); - try { - for (const sequence of [1, 2]) { - const response = await fetch(`${server.url}/logs3`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ sequence, api_version: 2, rows: [] }), - }); - await response.text(); + it.each(["/logs3", "/otel/v1/traces"])( + "retries transient failures for %s before forwarding later writes", + async (path) => { + const received: Array<{ sequence: number; rows: unknown[] }> = []; + let attempts = 0; + const upstream = createServer(async (req, res) => { + let body = ""; + for await (const chunk of req) body += chunk; + const payload = JSON.parse(body); + received.push(payload); + // Reproduce a gateway outage followed by successful ingestion. + res.statusCode = payload.sequence === 1 && ++attempts <= 2 ? 502 : 200; + res.end(res.statusCode === 502 ? "Bad Gateway" : "{}"); + }); + await new Promise((resolve) => + upstream.listen(0, "127.0.0.1", resolve), + ); + const url = `http://127.0.0.1:${(upstream.address() as AddressInfo).port}`; + const server = await startMockBraintrustServer({ + prodForwarding: { + apiKey: "test-only-key", + apiUrl: url, + appUrl: url, + orgId: "org", + orgName: "org", + projectId: "project", + projectName: "tmp-luca-forwarding-test", + }, + }); + try { + for (const sequence of [1, 2]) { + const response = await fetch(`${server.url}${path}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + sequence, + api_version: 2, + rows: [{ id: "span", _is_merge: sequence === 2 }], + }), + }); + expect(response.ok).toBe(true); + await response.text(); + } + await expect(server.close()).resolves.toBeUndefined(); + expect(received.map((payload) => payload.sequence)).toEqual([ + 1, 1, 1, 2, + ]); + expect(received[1]).toEqual(received[0]); + expect(received[2]).toEqual(received[0]); + } finally { + upstream.closeAllConnections(); + await new Promise((resolve) => upstream.close(() => resolve())); } - await expect(server.close()).rejects.toThrow("initial write failed"); - expect(received).toEqual([1, 2]); - } finally { - upstream.closeAllConnections(); - await new Promise((resolve) => upstream.close(() => resolve())); - } - }); + }, + ); + + it.each([ + { status: 400, expectedSequence: [1, 2] }, + { status: 401, expectedSequence: [1, 2] }, + { status: 500, expectedSequence: [1, 1, 1, 2] }, + { status: 502, expectedSequence: [1, 1, 1, 2] }, + { status: 503, expectedSequence: [1, 1, 1, 2] }, + { status: 504, expectedSequence: [1, 1, 1, 2] }, + ])( + "reports HTTP $status failures without preventing later queued writes", + async ({ status, expectedSequence }) => { + const received: number[] = []; + const upstream = createServer(async (req, res) => { + let body = ""; + for await (const chunk of req) body += chunk; + const { sequence } = JSON.parse(body); + received.push(sequence); + res.statusCode = sequence === 1 ? status : 200; + res.end(sequence === 1 ? "initial write failed" : "{}"); + }); + await new Promise((resolve) => + upstream.listen(0, "127.0.0.1", resolve), + ); + const url = `http://127.0.0.1:${(upstream.address() as AddressInfo).port}`; + const server = await startMockBraintrustServer({ + prodForwarding: { + apiKey: "test-only-key", + apiUrl: url, + appUrl: url, + orgId: "org", + orgName: "org", + projectId: "project", + projectName: "tmp-luca-forwarding-test", + }, + }); + try { + for (const sequence of [1, 2]) { + const response = await fetch(`${server.url}/logs3`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sequence, api_version: 2, rows: [] }), + }); + await response.text(); + } + await expect(server.close()).rejects.toThrow("initial write failed"); + expect(received).toEqual(expectedSequence); + } finally { + upstream.closeAllConnections(); + await new Promise((resolve) => upstream.close(() => resolve())); + } + }, + ); }); diff --git a/e2e/helpers/mock-braintrust-server.ts b/e2e/helpers/mock-braintrust-server.ts index 14a42dbc3..6713934d9 100644 --- a/e2e/helpers/mock-braintrust-server.ts +++ b/e2e/helpers/mock-braintrust-server.ts @@ -6,6 +6,7 @@ import type { ServerResponse, } from "node:http"; import type { AddressInfo } from "node:net"; +import { setTimeout } from "node:timers/promises"; import type { ProdForwarding } from "./prod-forwarding"; export type JsonValue = @@ -470,14 +471,31 @@ export async function startMockBraintrustServer( } headers.set("authorization", `Bearer ${prodForwarding.apiKey}`); - const response = await fetch(url, { + const requestInit = { body: prodRequest.method === "GET" || prodRequest.method === "HEAD" ? undefined : prodRequest.rawBody, headers, method: prodRequest.method, - }); + }; + // Retry ingestion inside the forwarding queue so later merges cannot + // overtake the initial upsert. Registration requests remain single-attempt. + const maxAttempts = + prodRequest.method === "POST" && + ["/logs3", "/otel/v1/traces"].includes(prodRequest.path) + ? 3 + : 1; + let response = await fetch(url, requestInit); + for ( + let attempt = 1; + attempt < maxAttempts && [500, 502, 503, 504].includes(response.status); + attempt++ + ) { + await response.arrayBuffer().catch(() => {}); + await setTimeout(500 * 2 ** (attempt - 1)); + response = await fetch(url, requestInit); + } if (!response.ok) { const responseText = await response.text().catch(() => ""); diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v2-latest.span-tree.json b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v2-latest.span-tree.json index fc850743b..e8fdf9647 100644 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v2-latest.span-tree.json +++ b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v2-latest.span-tree.json @@ -230,7 +230,8 @@ "children": [], "output": "lookup-active", "metadata": { - "scenario": "flue-instrumentation" + "scenario": "flue-instrumentation", + "testRunId": "" }, "context": { "caller_filename": "/e2e/scenarios/flue-instrumentation/scenario.v2.ts", diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v2-latest.span-tree.txt b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v2-latest.span-tree.txt index 89dfaabc1..460ccd5a5 100644 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v2-latest.span-tree.txt +++ b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v2-latest.span-tree.txt @@ -277,7 +277,8 @@ span_tree: └── flue.toolCurrentProbe output: "lookup-active" metadata: { - "scenario": "flue-instrumentation" + "scenario": "flue-instrumentation", + "testRunId": "" } context: { "caller_filename": "/e2e/scenarios/flue-instrumentation/scenario.v2.ts", diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v2.span-tree.json b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v2.span-tree.json index c152b398e..93addbebe 100644 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v2.span-tree.json +++ b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v2.span-tree.json @@ -230,7 +230,8 @@ "children": [], "output": "lookup-active", "metadata": { - "scenario": "flue-instrumentation" + "scenario": "flue-instrumentation", + "testRunId": "" }, "context": { "caller_filename": "/e2e/scenarios/flue-instrumentation/scenario.v2.ts", diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v2.span-tree.txt b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v2.span-tree.txt index b6bb60e0c..79190c338 100644 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v2.span-tree.txt +++ b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v2.span-tree.txt @@ -277,7 +277,8 @@ span_tree: └── flue.toolCurrentProbe output: "lookup-active" metadata: { - "scenario": "flue-instrumentation" + "scenario": "flue-instrumentation", + "testRunId": "" } context: { "caller_filename": "/e2e/scenarios/flue-instrumentation/scenario.v2.ts", diff --git a/e2e/scenarios/flue-instrumentation/scenario.v2.ts b/e2e/scenarios/flue-instrumentation/scenario.v2.ts index 567d47321..b45091536 100644 --- a/e2e/scenarios/flue-instrumentation/scenario.v2.ts +++ b/e2e/scenarios/flue-instrumentation/scenario.v2.ts @@ -33,7 +33,12 @@ const lookup = defineTool({ }, { name: "flue.toolCurrentProbe", - event: { metadata: { scenario: SCENARIO_NAME } }, + event: { + metadata: { + scenario: SCENARIO_NAME, + testRunId: process.env.BRAINTRUST_E2E_RUN_ID, + }, + }, }, ); return { diff --git a/e2e/scenarios/flue-instrumentation/v2-assertions.ts b/e2e/scenarios/flue-instrumentation/v2-assertions.ts index e4e9b9310..92f69426f 100644 --- a/e2e/scenarios/flue-instrumentation/v2-assertions.ts +++ b/e2e/scenarios/flue-instrumentation/v2-assertions.ts @@ -36,9 +36,11 @@ export function defineFlueV2InstrumentationAssertions(options: { describe.sequential("explicit native instrumentation", () => { let events: CapturedLogEvent[] = []; + let testRunId: string; beforeAll(async () => { await withScenarioHarness(async (harness) => { + testRunId = harness.testRunId; await harness.runScenarioDir({ entry: "scenario.v2.ts", env: { @@ -121,6 +123,10 @@ export function defineFlueV2InstrumentationAssertions(options: { }); expect(probe?.span.parentIds).toEqual([tool?.span.id]); expect(probe?.output).toBe("lookup-active"); + expect(probe?.metadata).toMatchObject({ + scenario: "flue-instrumentation", + testRunId, + }); }); test("matches the Flue 2 span tree snapshot", testConfig, async () => {