From 3e85dd47fb063859c46d2f50789936cfcb50b04c Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Thu, 17 Sep 2026 09:45:42 +0000 Subject: [PATCH 1/4] feat: Add instrumentation for `@typesafe-ai/sdk` --- e2e/README.md | 9 +- e2e/config/pr-comment-scenarios.json | 15 + e2e/helpers/scenario-harness.ts | 3 + .../typesafe-v0-latest.cassette.json | 154 ++++++++++ .../__cassettes__/typesafe-v0.cassette.json | 154 ++++++++++ .../typesafe-v0-latest-wrapped.span-tree.json | 137 +++++++++ .../typesafe-v0-latest-wrapped.span-tree.txt | 113 ++++++++ .../typesafe-v0-latest.span-tree.json | 137 +++++++++ .../typesafe-v0-latest.span-tree.txt | 113 ++++++++ .../typesafe-v0-wrapped.span-tree.json | 137 +++++++++ .../typesafe-v0-wrapped.span-tree.txt | 113 ++++++++ .../__snapshots__/typesafe-v0.span-tree.json | 137 +++++++++ .../__snapshots__/typesafe-v0.span-tree.txt | 113 ++++++++ .../typesafe-instrumentation/assertions.ts | 138 +++++++++ .../typesafe-instrumentation/package.json | 18 ++ .../typesafe-instrumentation/pnpm-lock.yaml | 26 ++ .../scenario.cjs-client.mjs | 9 + .../scenario.cjs-client.ts | 10 + .../scenario.impl.mjs | 90 ++++++ .../typesafe-instrumentation/scenario.mjs | 7 + .../typesafe-instrumentation/scenario.test.ts | 76 +++++ .../typesafe-instrumentation/scenario.ts | 9 + e2e/scripts/run-e2e-bump-docker.mjs | 2 + js/src/auto-instrumentations/configs/all.ts | 5 + .../auto-instrumentations/configs/typesafe.ts | 19 ++ js/src/exports.ts | 1 + .../instrumentation/braintrust-plugin.test.ts | 23 ++ js/src/instrumentation/braintrust-plugin.ts | 12 + js/src/instrumentation/config.ts | 5 + .../plugins/typesafe-channels.ts | 24 ++ .../plugins/typesafe-plugin.ts | 271 ++++++++++++++++++ js/src/span-origin.ts | 1 + js/src/vendor-sdk-types/typesafe.ts | 44 +++ js/src/wrappers/typesafe.test.ts | 235 +++++++++++++++ js/src/wrappers/typesafe.ts | 48 ++++ 35 files changed, 2404 insertions(+), 4 deletions(-) create mode 100644 e2e/scenarios/typesafe-instrumentation/__cassettes__/typesafe-v0-latest.cassette.json create mode 100644 e2e/scenarios/typesafe-instrumentation/__cassettes__/typesafe-v0.cassette.json create mode 100644 e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest-wrapped.span-tree.json create mode 100644 e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest-wrapped.span-tree.txt create mode 100644 e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest.span-tree.json create mode 100644 e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest.span-tree.txt create mode 100644 e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-wrapped.span-tree.json create mode 100644 e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-wrapped.span-tree.txt create mode 100644 e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0.span-tree.json create mode 100644 e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0.span-tree.txt create mode 100644 e2e/scenarios/typesafe-instrumentation/assertions.ts create mode 100644 e2e/scenarios/typesafe-instrumentation/package.json create mode 100644 e2e/scenarios/typesafe-instrumentation/pnpm-lock.yaml create mode 100644 e2e/scenarios/typesafe-instrumentation/scenario.cjs-client.mjs create mode 100644 e2e/scenarios/typesafe-instrumentation/scenario.cjs-client.ts create mode 100644 e2e/scenarios/typesafe-instrumentation/scenario.impl.mjs create mode 100644 e2e/scenarios/typesafe-instrumentation/scenario.mjs create mode 100644 e2e/scenarios/typesafe-instrumentation/scenario.test.ts create mode 100644 e2e/scenarios/typesafe-instrumentation/scenario.ts create mode 100644 js/src/auto-instrumentations/configs/typesafe.ts create mode 100644 js/src/instrumentation/plugins/typesafe-channels.ts create mode 100644 js/src/instrumentation/plugins/typesafe-plugin.ts create mode 100644 js/src/vendor-sdk-types/typesafe.ts create mode 100644 js/src/wrappers/typesafe.test.ts create mode 100644 js/src/wrappers/typesafe.ts diff --git a/e2e/README.md b/e2e/README.md index d9ac582ea..f1ebcbecb 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -109,6 +109,7 @@ Provider credentials are only required when recording or explicitly running live - `HUGGINGFACE_API_KEY` - `COHERE_API_KEY` - `GROQ_API_KEY` +- `TYPESAFE_API_KEY` `claude-agent-sdk-instrumentation` also uses `ANTHROPIC_API_KEY`, because it runs the real Claude Agent SDK against Anthropic in the same style as the existing live Anthropic wrapper coverage. @@ -142,7 +143,7 @@ The mock Braintrust server captures **outbound** SDK→Braintrust traffic and sn The cassette layer is backed by the internal `@braintrust/seinfeld` workspace package. For e2e tests, the harness starts a local cassette HTTP server and points provider SDK base URLs at that server. -- **Layer:** `withScenarioHarness(...)` starts a `createCassetteServer()` instance for cassette-enabled scenario runs. Provider base URL env vars (for OpenAI, Anthropic, Anthropic Bedrock, Google, Cohere, Cursor, Groq, HuggingFace, Mistral, OpenRouter, and the GitHub Copilot SDK's OpenAI provider mode) point at route prefixes on that server, so subprocesses and SDK-launched binaries can be captured too. +- **Layer:** `withScenarioHarness(...)` starts a `createCassetteServer()` instance for cassette-enabled scenario runs. Provider base URL env vars (for OpenAI, Anthropic, Anthropic Bedrock, Google, Cohere, Cursor, Groq, HuggingFace, Mistral, OpenRouter, TypeSafe, and the GitHub Copilot SDK's OpenAI provider mode) point at route prefixes on that server, so subprocesses and SDK-launched binaries can be captured too. - **Auto-engage:** the harness automatically engages the cassette layer when (a) a scenario run has `runContext.variantKey`, (b) a cassette JSON exists for the scenario+variant on disk, OR (c) `BRAINTRUST_E2E_CASSETTE_MODE` is `record` / `record-missing`. Scenarios just need to thread `runContext: { variantKey, originalScenarioDir }` into their runner calls — no other code change required. - **Provider replay failures:** provider scenarios are not skipped when a cassette is missing. In replay mode, the cassette server still starts from `runContext.variantKey`, injects placeholder provider keys, and fails on cassette misses instead of silently calling a live provider. - **Mode** is set by `BRAINTRUST_E2E_CASSETTE_MODE`: @@ -169,14 +170,14 @@ ANTHROPIC_API_KEY=... AWS_BEARER_TOKEN_BEDROCK=... \ OPENAI_API_KEY=... GEMINI_API_KEY=... \ COHERE_API_KEY=... GROQ_API_KEY=... HUGGINGFACE_API_KEY=... \ MISTRAL_API_KEY=... OLLAMA_API_KEY=... OPENROUTER_API_KEY=... \ -CURSOR_API_KEY=... \ +CURSOR_API_KEY=... TYPESAFE_API_KEY=... \ pnpm --filter=@braintrust/js-e2e-tests run test:e2e:record ``` After recording, run again **without any provider keys** to confirm the cassette is sufficient: ```bash -unset ANTHROPIC_API_KEY AWS_BEARER_TOKEN_BEDROCK OPENAI_API_KEY GEMINI_API_KEY GOOGLE_API_KEY GOOGLE_GENAI_API_KEY COHERE_API_KEY GROQ_API_KEY HUGGINGFACE_API_KEY MISTRAL_API_KEY OLLAMA_API_KEY OPENROUTER_API_KEY CURSOR_API_KEY +unset ANTHROPIC_API_KEY AWS_BEARER_TOKEN_BEDROCK OPENAI_API_KEY GEMINI_API_KEY GOOGLE_API_KEY GOOGLE_GENAI_API_KEY COHERE_API_KEY GROQ_API_KEY HUGGINGFACE_API_KEY MISTRAL_API_KEY OLLAMA_API_KEY OPENROUTER_API_KEY CURSOR_API_KEY TYPESAFE_API_KEY pnpm --filter=@braintrust/js-e2e-tests run test:e2e ``` @@ -188,7 +189,7 @@ After any successful record run, stale cassette variants are cleaned only inside These scenarios have cassette wiring in place and will use cassettes once they're recorded: -`anthropic-bedrock-instrumentation`, `anthropic-instrumentation`, `openai-instrumentation`, `openai-codex-instrumentation`, `ai-sdk-instrumentation`, `ai-sdk-otel-export`, `claude-agent-sdk-instrumentation`, `cohere-instrumentation`, `cursor-sdk-instrumentation`, `github-copilot-instrumentation`, `google-adk-instrumentation`, `google-genai-instrumentation`, `groq-instrumentation`, `huggingface-instrumentation`, `mistral-instrumentation`, `ollama-instrumentation`, `openrouter-agent-instrumentation`, `openrouter-instrumentation`, `wrap-langchain-js-traces`. +`anthropic-bedrock-instrumentation`, `anthropic-instrumentation`, `openai-instrumentation`, `openai-codex-instrumentation`, `ai-sdk-instrumentation`, `ai-sdk-otel-export`, `claude-agent-sdk-instrumentation`, `cohere-instrumentation`, `cursor-sdk-instrumentation`, `github-copilot-instrumentation`, `google-adk-instrumentation`, `google-genai-instrumentation`, `groq-instrumentation`, `huggingface-instrumentation`, `mistral-instrumentation`, `ollama-instrumentation`, `openrouter-agent-instrumentation`, `openrouter-instrumentation`, `typesafe-instrumentation`, `wrap-langchain-js-traces`. ### Cassette format diff --git a/e2e/config/pr-comment-scenarios.json b/e2e/config/pr-comment-scenarios.json index 59ef51510..0540d6c3a 100644 --- a/e2e/config/pr-comment-scenarios.json +++ b/e2e/config/pr-comment-scenarios.json @@ -509,6 +509,21 @@ } ] }, + { + "scenarioDirName": "typesafe-instrumentation", + "label": "TypeSafe Instrumentation", + "metadataScenario": "typesafe-instrumentation", + "variants": [ + { + "variantKey": "typesafe-v0", + "label": "v0 pinned" + }, + { + "variantKey": "typesafe-v0-latest", + "label": "v0 latest" + } + ] + }, { "scenarioDirName": "openrouter-instrumentation", "label": "OpenRouter Instrumentation", diff --git a/e2e/helpers/scenario-harness.ts b/e2e/helpers/scenario-harness.ts index 0816f0414..b86753c4a 100644 --- a/e2e/helpers/scenario-harness.ts +++ b/e2e/helpers/scenario-harness.ts @@ -320,6 +320,7 @@ function getCassetteServerRoutes(): CassetteServerRoute[] { { prefix: "/ollama", upstreamOrigin: "https://ollama.com" }, { prefix: "/openai", upstreamOrigin: "https://api.openai.com" }, { prefix: "/openrouter", upstreamOrigin: "https://openrouter.ai" }, + { prefix: "/typesafe", upstreamOrigin: "https://api.typesafe.ai" }, ]; } @@ -356,6 +357,7 @@ function getCassetteEnv(wiring: ActiveCassetteWiring): Record { OLLAMA_HOST: `${serverUrl}/ollama`, OPENAI_BASE_URL: `${serverUrl}/openai/v1`, OPENROUTER_BASE_URL: `${serverUrl}/openrouter/api/v1`, + TYPESAFE_BASE_URL: `${serverUrl}/typesafe`, }; } @@ -410,6 +412,7 @@ const CASSETTE_PROVIDER_KEYS: Array<{ envVars: ["OPENROUTER_API_KEY"], placeholder: "sk-or-cassette-placeholder", }, + { envVars: ["TYPESAFE_API_KEY"], placeholder: "cassette-placeholder" }, ]; function getProviderKeyPlaceholders(): Record { diff --git a/e2e/scenarios/typesafe-instrumentation/__cassettes__/typesafe-v0-latest.cassette.json b/e2e/scenarios/typesafe-instrumentation/__cassettes__/typesafe-v0-latest.cassette.json new file mode 100644 index 000000000..cc71fa45a --- /dev/null +++ b/e2e/scenarios/typesafe-instrumentation/__cassettes__/typesafe-v0-latest.cassette.json @@ -0,0 +1,154 @@ +{ + "entries": [ + { + "callIndex": 0, + "id": "c89a379b9929505d", + "matchKey": "POST api.typesafe.ai/v1/systemone", + "recordedAt": "2026-09-17T09:28:46.455Z", + "request": { + "body": { + "kind": "json", + "value": { + "model": "jev-latest", + "questions": { + "category": { + "criteria": { + "billing": null, + "other": null, + "technical": null + }, + "instructions": "Which team should handle this?", + "type": "choice" + }, + "duplicate_charge": { + "instructions": { + "question": "Does the customer report a duplicate charge?" + }, + "type": "noul" + }, + "urgency": { + "criteria": ["routine", "soon", "urgent"], + "instructions": "How urgent is this request?", + "type": "score" + } + }, + "state": { + "message": "I was charged twice. Please refund the duplicate charge today." + } + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.typesafe.ai/v1/systemone" + }, + "response": { + "body": { + "kind": "json", + "value": { + "answers": { + "category": { + "choice": "billing", + "confidence": 1, + "probabilities": { + "billing": 1, + "other": 0, + "technical": 0 + }, + "type": "choice" + }, + "duplicate_charge": { + "noul": 0.99, + "type": "noul" + }, + "urgency": { + "confidence": 0.66, + "legend": { + "0": "routine", + "1": "soon", + "2": "urgent" + }, + "probabilities": { + "0": 0, + "1": 0.22, + "2": 0.78 + }, + "score": 1.78, + "type": "score" + } + }, + "model": "jev-1.13.0", + "usage": { + "input_tokens": 370, + "output_tokens": 70 + } + } + }, + "headers": { + "content-length": "405", + "content-type": "application/json", + "date": "Thu, 17 Sep 2026 09:28:45 GMT", + "server": "istio-envoy", + "x-envoy-upstream-service-time": "276", + "x-typesafe-request-id": "req_01a0aeb275447b61b959098407b287b8" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 1, + "id": "f496f1d9a9324031", + "matchKey": "POST api.typesafe.ai/v1/systemone", + "recordedAt": "2026-09-17T09:28:46.630Z", + "request": { + "body": { + "kind": "json", + "value": { + "model": "jev-1.13.0", + "questions": { + "positive": { + "instructions": "Is this feedback positive?", + "type": "noul" + } + }, + "state": "The package arrived intact and on time." + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.typesafe.ai/v1/systemone" + }, + "response": { + "body": { + "kind": "json", + "value": { + "answers": { + "positive": { + "noul": 0.97, + "type": "noul" + } + }, + "model": "jev-1.13.0", + "usage": { + "input_tokens": 279, + "output_tokens": 20 + } + } + }, + "headers": { + "content-length": "121", + "content-type": "application/json", + "date": "Thu, 17 Sep 2026 09:28:46 GMT", + "server": "istio-envoy", + "x-envoy-upstream-service-time": "110", + "x-typesafe-request-id": "req_01a0aeb2769b7c5a802b5f938375ad4b" + }, + "status": 200, + "statusText": "OK" + } + } + ], + "meta": { + "createdAt": "2026-09-17T09:28:45.359Z" + } +} diff --git a/e2e/scenarios/typesafe-instrumentation/__cassettes__/typesafe-v0.cassette.json b/e2e/scenarios/typesafe-instrumentation/__cassettes__/typesafe-v0.cassette.json new file mode 100644 index 000000000..b75a11cdc --- /dev/null +++ b/e2e/scenarios/typesafe-instrumentation/__cassettes__/typesafe-v0.cassette.json @@ -0,0 +1,154 @@ +{ + "entries": [ + { + "callIndex": 0, + "id": "c89a379b9929505d", + "matchKey": "POST api.typesafe.ai/v1/systemone", + "recordedAt": "2026-09-17T09:28:43.925Z", + "request": { + "body": { + "kind": "json", + "value": { + "model": "jev-latest", + "questions": { + "category": { + "criteria": { + "billing": null, + "other": null, + "technical": null + }, + "instructions": "Which team should handle this?", + "type": "choice" + }, + "duplicate_charge": { + "instructions": { + "question": "Does the customer report a duplicate charge?" + }, + "type": "noul" + }, + "urgency": { + "criteria": ["routine", "soon", "urgent"], + "instructions": "How urgent is this request?", + "type": "score" + } + }, + "state": { + "message": "I was charged twice. Please refund the duplicate charge today." + } + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.typesafe.ai/v1/systemone" + }, + "response": { + "body": { + "kind": "json", + "value": { + "answers": { + "category": { + "choice": "billing", + "confidence": 1, + "probabilities": { + "billing": 1, + "other": 0, + "technical": 0 + }, + "type": "choice" + }, + "duplicate_charge": { + "noul": 0.99, + "type": "noul" + }, + "urgency": { + "confidence": 0.71, + "legend": { + "0": "routine", + "1": "soon", + "2": "urgent" + }, + "probabilities": { + "0": 0, + "1": 0.19, + "2": 0.81 + }, + "score": 1.8, + "type": "score" + } + }, + "model": "jev-1.13.0", + "usage": { + "input_tokens": 370, + "output_tokens": 70 + } + } + }, + "headers": { + "content-length": "404", + "content-type": "application/json", + "date": "Thu, 17 Sep 2026 09:28:43 GMT", + "server": "istio-envoy", + "x-envoy-upstream-service-time": "117", + "x-typesafe-request-id": "req_01a0aeb26c037ff9bbe4e12634b1e4dc" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 1, + "id": "f496f1d9a9324031", + "matchKey": "POST api.typesafe.ai/v1/systemone", + "recordedAt": "2026-09-17T09:28:44.134Z", + "request": { + "body": { + "kind": "json", + "value": { + "model": "jev-1.13.0", + "questions": { + "positive": { + "instructions": "Is this feedback positive?", + "type": "noul" + } + }, + "state": "The package arrived intact and on time." + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.typesafe.ai/v1/systemone" + }, + "response": { + "body": { + "kind": "json", + "value": { + "answers": { + "positive": { + "noul": 0.97, + "type": "noul" + } + }, + "model": "jev-1.13.0", + "usage": { + "input_tokens": 279, + "output_tokens": 20 + } + } + }, + "headers": { + "content-length": "121", + "content-type": "application/json", + "date": "Thu, 17 Sep 2026 09:28:43 GMT", + "server": "istio-envoy", + "x-envoy-upstream-service-time": "144", + "x-typesafe-request-id": "req_01a0aeb26cb77f01bafc3f74fcb09908" + }, + "status": 200, + "statusText": "OK" + } + } + ], + "meta": { + "createdAt": "2026-09-17T09:28:42.423Z" + } +} diff --git a/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest-wrapped.span-tree.json b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest-wrapped.span-tree.json new file mode 100644 index 000000000..8c3e187d2 --- /dev/null +++ b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest-wrapped.span-tree.json @@ -0,0 +1,137 @@ +{ + "span_tree": [ + { + "name": "typesafe-instrumentation-root", + "type": "task", + "children": [ + { + "name": "typesafe-mixed-operation", + "children": [ + { + "name": "typesafe.systemOne", + "type": "llm", + "children": [], + "input": { + "questions": { + "category": { + "criteria": { + "billing": null, + "other": null, + "technical": null + }, + "instructions": "Which team should handle this?", + "type": "choice" + }, + "duplicate_charge": { + "instructions": { + "question": "Does the customer report a duplicate charge?" + }, + "type": "noul" + }, + "urgency": { + "criteria": [ + "routine", + "soon", + "urgent" + ], + "instructions": "How urgent is this request?", + "type": "score" + } + }, + "state": { + "message": "I was charged twice. Please refund the duplicate charge today." + } + }, + "output": { + "category": { + "choice": "billing", + "confidence": 1, + "probabilities": { + "billing": 1, + "other": 0, + "technical": 0 + }, + "type": "choice" + }, + "duplicate_charge": { + "noul": 0.99, + "type": "noul" + }, + "urgency": { + "confidence": 0.66, + "legend": { + "0": "routine", + "1": "soon", + "2": "urgent" + }, + "probabilities": { + "0": 0, + "1": 0.22, + "2": 0.78 + }, + "score": 1.78, + "type": "score" + } + }, + "metadata": { + "model": "jev-1.13.0", + "provider": "typesafe" + }, + "metrics": { + "completion_tokens": 70, + "prompt_tokens": 370, + "tokens": 440 + } + } + ], + "metadata": { + "operation": "mixed", + "testRunId": "" + } + }, + { + "name": "typesafe-raw-operation", + "children": [ + { + "name": "typesafe.systemOne", + "type": "llm", + "children": [], + "input": { + "questions": { + "positive": { + "instructions": "Is this feedback positive?", + "type": "noul" + } + }, + "state": "The package arrived intact and on time." + }, + "output": { + "positive": { + "noul": 0.97, + "type": "noul" + } + }, + "metadata": { + "model": "jev-1.13.0", + "provider": "typesafe" + }, + "metrics": { + "completion_tokens": 20, + "prompt_tokens": 279, + "tokens": 299 + } + } + ], + "metadata": { + "operation": "raw", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "typesafe-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest-wrapped.span-tree.txt b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest-wrapped.span-tree.txt new file mode 100644 index 000000000..1e4a6baac --- /dev/null +++ b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest-wrapped.span-tree.txt @@ -0,0 +1,113 @@ +span_tree: +└── typesafe-instrumentation-root [task] + metadata: { + "scenario": "typesafe-instrumentation", + "testRunId": "" + } + ├── typesafe-mixed-operation + │ metadata: { + │ "operation": "mixed", + │ "testRunId": "" + │ } + │ └── typesafe.systemOne [llm] + │ input: { + │ "questions": { + │ "category": { + │ "criteria": { + │ "billing": null, + │ "other": null, + │ "technical": null + │ }, + │ "instructions": "Which team should handle this?", + │ "type": "choice" + │ }, + │ "duplicate_charge": { + │ "instructions": { + │ "question": "Does the customer report a duplicate charge?" + │ }, + │ "type": "noul" + │ }, + │ "urgency": { + │ "criteria": [ + │ "routine", + │ "soon", + │ "urgent" + │ ], + │ "instructions": "How urgent is this request?", + │ "type": "score" + │ } + │ }, + │ "state": { + │ "message": "I was charged twice. Please refund the duplicate charge today." + │ } + │ } + │ output: { + │ "category": { + │ "choice": "billing", + │ "confidence": 1, + │ "probabilities": { + │ "billing": 1, + │ "other": 0, + │ "technical": 0 + │ }, + │ "type": "choice" + │ }, + │ "duplicate_charge": { + │ "noul": 0.99, + │ "type": "noul" + │ }, + │ "urgency": { + │ "confidence": 0.66, + │ "legend": { + │ "0": "routine", + │ "1": "soon", + │ "2": "urgent" + │ }, + │ "probabilities": { + │ "0": 0, + │ "1": 0.22, + │ "2": 0.78 + │ }, + │ "score": 1.78, + │ "type": "score" + │ } + │ } + │ metadata: { + │ "model": "jev-1.13.0", + │ "provider": "typesafe" + │ } + │ metrics: { + │ "completion_tokens": 70, + │ "prompt_tokens": 370, + │ "tokens": 440 + │ } + └── typesafe-raw-operation + metadata: { + "operation": "raw", + "testRunId": "" + } + └── typesafe.systemOne [llm] + input: { + "questions": { + "positive": { + "instructions": "Is this feedback positive?", + "type": "noul" + } + }, + "state": "The package arrived intact and on time." + } + output: { + "positive": { + "noul": 0.97, + "type": "noul" + } + } + metadata: { + "model": "jev-1.13.0", + "provider": "typesafe" + } + metrics: { + "completion_tokens": 20, + "prompt_tokens": 279, + "tokens": 299 + } diff --git a/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest.span-tree.json b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest.span-tree.json new file mode 100644 index 000000000..8c3e187d2 --- /dev/null +++ b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest.span-tree.json @@ -0,0 +1,137 @@ +{ + "span_tree": [ + { + "name": "typesafe-instrumentation-root", + "type": "task", + "children": [ + { + "name": "typesafe-mixed-operation", + "children": [ + { + "name": "typesafe.systemOne", + "type": "llm", + "children": [], + "input": { + "questions": { + "category": { + "criteria": { + "billing": null, + "other": null, + "technical": null + }, + "instructions": "Which team should handle this?", + "type": "choice" + }, + "duplicate_charge": { + "instructions": { + "question": "Does the customer report a duplicate charge?" + }, + "type": "noul" + }, + "urgency": { + "criteria": [ + "routine", + "soon", + "urgent" + ], + "instructions": "How urgent is this request?", + "type": "score" + } + }, + "state": { + "message": "I was charged twice. Please refund the duplicate charge today." + } + }, + "output": { + "category": { + "choice": "billing", + "confidence": 1, + "probabilities": { + "billing": 1, + "other": 0, + "technical": 0 + }, + "type": "choice" + }, + "duplicate_charge": { + "noul": 0.99, + "type": "noul" + }, + "urgency": { + "confidence": 0.66, + "legend": { + "0": "routine", + "1": "soon", + "2": "urgent" + }, + "probabilities": { + "0": 0, + "1": 0.22, + "2": 0.78 + }, + "score": 1.78, + "type": "score" + } + }, + "metadata": { + "model": "jev-1.13.0", + "provider": "typesafe" + }, + "metrics": { + "completion_tokens": 70, + "prompt_tokens": 370, + "tokens": 440 + } + } + ], + "metadata": { + "operation": "mixed", + "testRunId": "" + } + }, + { + "name": "typesafe-raw-operation", + "children": [ + { + "name": "typesafe.systemOne", + "type": "llm", + "children": [], + "input": { + "questions": { + "positive": { + "instructions": "Is this feedback positive?", + "type": "noul" + } + }, + "state": "The package arrived intact and on time." + }, + "output": { + "positive": { + "noul": 0.97, + "type": "noul" + } + }, + "metadata": { + "model": "jev-1.13.0", + "provider": "typesafe" + }, + "metrics": { + "completion_tokens": 20, + "prompt_tokens": 279, + "tokens": 299 + } + } + ], + "metadata": { + "operation": "raw", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "typesafe-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest.span-tree.txt b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest.span-tree.txt new file mode 100644 index 000000000..1e4a6baac --- /dev/null +++ b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest.span-tree.txt @@ -0,0 +1,113 @@ +span_tree: +└── typesafe-instrumentation-root [task] + metadata: { + "scenario": "typesafe-instrumentation", + "testRunId": "" + } + ├── typesafe-mixed-operation + │ metadata: { + │ "operation": "mixed", + │ "testRunId": "" + │ } + │ └── typesafe.systemOne [llm] + │ input: { + │ "questions": { + │ "category": { + │ "criteria": { + │ "billing": null, + │ "other": null, + │ "technical": null + │ }, + │ "instructions": "Which team should handle this?", + │ "type": "choice" + │ }, + │ "duplicate_charge": { + │ "instructions": { + │ "question": "Does the customer report a duplicate charge?" + │ }, + │ "type": "noul" + │ }, + │ "urgency": { + │ "criteria": [ + │ "routine", + │ "soon", + │ "urgent" + │ ], + │ "instructions": "How urgent is this request?", + │ "type": "score" + │ } + │ }, + │ "state": { + │ "message": "I was charged twice. Please refund the duplicate charge today." + │ } + │ } + │ output: { + │ "category": { + │ "choice": "billing", + │ "confidence": 1, + │ "probabilities": { + │ "billing": 1, + │ "other": 0, + │ "technical": 0 + │ }, + │ "type": "choice" + │ }, + │ "duplicate_charge": { + │ "noul": 0.99, + │ "type": "noul" + │ }, + │ "urgency": { + │ "confidence": 0.66, + │ "legend": { + │ "0": "routine", + │ "1": "soon", + │ "2": "urgent" + │ }, + │ "probabilities": { + │ "0": 0, + │ "1": 0.22, + │ "2": 0.78 + │ }, + │ "score": 1.78, + │ "type": "score" + │ } + │ } + │ metadata: { + │ "model": "jev-1.13.0", + │ "provider": "typesafe" + │ } + │ metrics: { + │ "completion_tokens": 70, + │ "prompt_tokens": 370, + │ "tokens": 440 + │ } + └── typesafe-raw-operation + metadata: { + "operation": "raw", + "testRunId": "" + } + └── typesafe.systemOne [llm] + input: { + "questions": { + "positive": { + "instructions": "Is this feedback positive?", + "type": "noul" + } + }, + "state": "The package arrived intact and on time." + } + output: { + "positive": { + "noul": 0.97, + "type": "noul" + } + } + metadata: { + "model": "jev-1.13.0", + "provider": "typesafe" + } + metrics: { + "completion_tokens": 20, + "prompt_tokens": 279, + "tokens": 299 + } diff --git a/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-wrapped.span-tree.json b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-wrapped.span-tree.json new file mode 100644 index 000000000..fe750162e --- /dev/null +++ b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-wrapped.span-tree.json @@ -0,0 +1,137 @@ +{ + "span_tree": [ + { + "name": "typesafe-instrumentation-root", + "type": "task", + "children": [ + { + "name": "typesafe-mixed-operation", + "children": [ + { + "name": "typesafe.systemOne", + "type": "llm", + "children": [], + "input": { + "questions": { + "category": { + "criteria": { + "billing": null, + "other": null, + "technical": null + }, + "instructions": "Which team should handle this?", + "type": "choice" + }, + "duplicate_charge": { + "instructions": { + "question": "Does the customer report a duplicate charge?" + }, + "type": "noul" + }, + "urgency": { + "criteria": [ + "routine", + "soon", + "urgent" + ], + "instructions": "How urgent is this request?", + "type": "score" + } + }, + "state": { + "message": "I was charged twice. Please refund the duplicate charge today." + } + }, + "output": { + "category": { + "choice": "billing", + "confidence": 1, + "probabilities": { + "billing": 1, + "other": 0, + "technical": 0 + }, + "type": "choice" + }, + "duplicate_charge": { + "noul": 0.99, + "type": "noul" + }, + "urgency": { + "confidence": 0.71, + "legend": { + "0": "routine", + "1": "soon", + "2": "urgent" + }, + "probabilities": { + "0": 0, + "1": 0.19, + "2": 0.81 + }, + "score": 1.8, + "type": "score" + } + }, + "metadata": { + "model": "jev-1.13.0", + "provider": "typesafe" + }, + "metrics": { + "completion_tokens": 70, + "prompt_tokens": 370, + "tokens": 440 + } + } + ], + "metadata": { + "operation": "mixed", + "testRunId": "" + } + }, + { + "name": "typesafe-raw-operation", + "children": [ + { + "name": "typesafe.systemOne", + "type": "llm", + "children": [], + "input": { + "questions": { + "positive": { + "instructions": "Is this feedback positive?", + "type": "noul" + } + }, + "state": "The package arrived intact and on time." + }, + "output": { + "positive": { + "noul": 0.97, + "type": "noul" + } + }, + "metadata": { + "model": "jev-1.13.0", + "provider": "typesafe" + }, + "metrics": { + "completion_tokens": 20, + "prompt_tokens": 279, + "tokens": 299 + } + } + ], + "metadata": { + "operation": "raw", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "typesafe-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-wrapped.span-tree.txt b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-wrapped.span-tree.txt new file mode 100644 index 000000000..1b38a474e --- /dev/null +++ b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-wrapped.span-tree.txt @@ -0,0 +1,113 @@ +span_tree: +└── typesafe-instrumentation-root [task] + metadata: { + "scenario": "typesafe-instrumentation", + "testRunId": "" + } + ├── typesafe-mixed-operation + │ metadata: { + │ "operation": "mixed", + │ "testRunId": "" + │ } + │ └── typesafe.systemOne [llm] + │ input: { + │ "questions": { + │ "category": { + │ "criteria": { + │ "billing": null, + │ "other": null, + │ "technical": null + │ }, + │ "instructions": "Which team should handle this?", + │ "type": "choice" + │ }, + │ "duplicate_charge": { + │ "instructions": { + │ "question": "Does the customer report a duplicate charge?" + │ }, + │ "type": "noul" + │ }, + │ "urgency": { + │ "criteria": [ + │ "routine", + │ "soon", + │ "urgent" + │ ], + │ "instructions": "How urgent is this request?", + │ "type": "score" + │ } + │ }, + │ "state": { + │ "message": "I was charged twice. Please refund the duplicate charge today." + │ } + │ } + │ output: { + │ "category": { + │ "choice": "billing", + │ "confidence": 1, + │ "probabilities": { + │ "billing": 1, + │ "other": 0, + │ "technical": 0 + │ }, + │ "type": "choice" + │ }, + │ "duplicate_charge": { + │ "noul": 0.99, + │ "type": "noul" + │ }, + │ "urgency": { + │ "confidence": 0.71, + │ "legend": { + │ "0": "routine", + │ "1": "soon", + │ "2": "urgent" + │ }, + │ "probabilities": { + │ "0": 0, + │ "1": 0.19, + │ "2": 0.81 + │ }, + │ "score": 1.8, + │ "type": "score" + │ } + │ } + │ metadata: { + │ "model": "jev-1.13.0", + │ "provider": "typesafe" + │ } + │ metrics: { + │ "completion_tokens": 70, + │ "prompt_tokens": 370, + │ "tokens": 440 + │ } + └── typesafe-raw-operation + metadata: { + "operation": "raw", + "testRunId": "" + } + └── typesafe.systemOne [llm] + input: { + "questions": { + "positive": { + "instructions": "Is this feedback positive?", + "type": "noul" + } + }, + "state": "The package arrived intact and on time." + } + output: { + "positive": { + "noul": 0.97, + "type": "noul" + } + } + metadata: { + "model": "jev-1.13.0", + "provider": "typesafe" + } + metrics: { + "completion_tokens": 20, + "prompt_tokens": 279, + "tokens": 299 + } diff --git a/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0.span-tree.json b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0.span-tree.json new file mode 100644 index 000000000..fe750162e --- /dev/null +++ b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0.span-tree.json @@ -0,0 +1,137 @@ +{ + "span_tree": [ + { + "name": "typesafe-instrumentation-root", + "type": "task", + "children": [ + { + "name": "typesafe-mixed-operation", + "children": [ + { + "name": "typesafe.systemOne", + "type": "llm", + "children": [], + "input": { + "questions": { + "category": { + "criteria": { + "billing": null, + "other": null, + "technical": null + }, + "instructions": "Which team should handle this?", + "type": "choice" + }, + "duplicate_charge": { + "instructions": { + "question": "Does the customer report a duplicate charge?" + }, + "type": "noul" + }, + "urgency": { + "criteria": [ + "routine", + "soon", + "urgent" + ], + "instructions": "How urgent is this request?", + "type": "score" + } + }, + "state": { + "message": "I was charged twice. Please refund the duplicate charge today." + } + }, + "output": { + "category": { + "choice": "billing", + "confidence": 1, + "probabilities": { + "billing": 1, + "other": 0, + "technical": 0 + }, + "type": "choice" + }, + "duplicate_charge": { + "noul": 0.99, + "type": "noul" + }, + "urgency": { + "confidence": 0.71, + "legend": { + "0": "routine", + "1": "soon", + "2": "urgent" + }, + "probabilities": { + "0": 0, + "1": 0.19, + "2": 0.81 + }, + "score": 1.8, + "type": "score" + } + }, + "metadata": { + "model": "jev-1.13.0", + "provider": "typesafe" + }, + "metrics": { + "completion_tokens": 70, + "prompt_tokens": 370, + "tokens": 440 + } + } + ], + "metadata": { + "operation": "mixed", + "testRunId": "" + } + }, + { + "name": "typesafe-raw-operation", + "children": [ + { + "name": "typesafe.systemOne", + "type": "llm", + "children": [], + "input": { + "questions": { + "positive": { + "instructions": "Is this feedback positive?", + "type": "noul" + } + }, + "state": "The package arrived intact and on time." + }, + "output": { + "positive": { + "noul": 0.97, + "type": "noul" + } + }, + "metadata": { + "model": "jev-1.13.0", + "provider": "typesafe" + }, + "metrics": { + "completion_tokens": 20, + "prompt_tokens": 279, + "tokens": 299 + } + } + ], + "metadata": { + "operation": "raw", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "typesafe-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0.span-tree.txt b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0.span-tree.txt new file mode 100644 index 000000000..1b38a474e --- /dev/null +++ b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0.span-tree.txt @@ -0,0 +1,113 @@ +span_tree: +└── typesafe-instrumentation-root [task] + metadata: { + "scenario": "typesafe-instrumentation", + "testRunId": "" + } + ├── typesafe-mixed-operation + │ metadata: { + │ "operation": "mixed", + │ "testRunId": "" + │ } + │ └── typesafe.systemOne [llm] + │ input: { + │ "questions": { + │ "category": { + │ "criteria": { + │ "billing": null, + │ "other": null, + │ "technical": null + │ }, + │ "instructions": "Which team should handle this?", + │ "type": "choice" + │ }, + │ "duplicate_charge": { + │ "instructions": { + │ "question": "Does the customer report a duplicate charge?" + │ }, + │ "type": "noul" + │ }, + │ "urgency": { + │ "criteria": [ + │ "routine", + │ "soon", + │ "urgent" + │ ], + │ "instructions": "How urgent is this request?", + │ "type": "score" + │ } + │ }, + │ "state": { + │ "message": "I was charged twice. Please refund the duplicate charge today." + │ } + │ } + │ output: { + │ "category": { + │ "choice": "billing", + │ "confidence": 1, + │ "probabilities": { + │ "billing": 1, + │ "other": 0, + │ "technical": 0 + │ }, + │ "type": "choice" + │ }, + │ "duplicate_charge": { + │ "noul": 0.99, + │ "type": "noul" + │ }, + │ "urgency": { + │ "confidence": 0.71, + │ "legend": { + │ "0": "routine", + │ "1": "soon", + │ "2": "urgent" + │ }, + │ "probabilities": { + │ "0": 0, + │ "1": 0.19, + │ "2": 0.81 + │ }, + │ "score": 1.8, + │ "type": "score" + │ } + │ } + │ metadata: { + │ "model": "jev-1.13.0", + │ "provider": "typesafe" + │ } + │ metrics: { + │ "completion_tokens": 70, + │ "prompt_tokens": 370, + │ "tokens": 440 + │ } + └── typesafe-raw-operation + metadata: { + "operation": "raw", + "testRunId": "" + } + └── typesafe.systemOne [llm] + input: { + "questions": { + "positive": { + "instructions": "Is this feedback positive?", + "type": "noul" + } + }, + "state": "The package arrived intact and on time." + } + output: { + "positive": { + "noul": 0.97, + "type": "noul" + } + } + metadata: { + "model": "jev-1.13.0", + "provider": "typesafe" + } + metrics: { + "completion_tokens": 20, + "prompt_tokens": 279, + "tokens": 299 + } diff --git a/e2e/scenarios/typesafe-instrumentation/assertions.ts b/e2e/scenarios/typesafe-instrumentation/assertions.ts new file mode 100644 index 000000000..9653a6576 --- /dev/null +++ b/e2e/scenarios/typesafe-instrumentation/assertions.ts @@ -0,0 +1,138 @@ +import { beforeAll, describe, expect, test } from "vitest"; +import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; +import { + withScenarioHarness, + type ScenarioRunContext, +} from "../../helpers/scenario-harness"; +import { matchSpanTreeSnapshot } from "../../helpers/span-tree"; +import { + findLatestChildSpan, + findLatestSpan, +} from "../../helpers/trace-selectors"; +import { ROOT_NAME, SCENARIO_NAME } from "./scenario.impl.mjs"; + +type RunTypeSafeScenario = (harness: { + runNodeScenarioDir: (options: { + entry: string; + env: Record; + nodeArgs: string[]; + runContext?: ScenarioRunContext; + scenarioDir: string; + }) => Promise; + runScenarioDir: (options: { + entry: string; + env: Record; + runContext?: ScenarioRunContext; + scenarioDir: string; + }) => Promise; +}) => Promise; + +const OPERATIONS = ["typesafe-mixed-operation", "typesafe-raw-operation"]; + +function spanTreeEvents(events: CapturedLogEvent[]): CapturedLogEvent[] { + return [ + findLatestSpan(events, ROOT_NAME), + ...OPERATIONS.flatMap((operationName) => { + const operation = findLatestSpan(events, operationName); + return [ + operation, + findLatestChildSpan(events, "typesafe.systemOne", operation?.span.id), + ]; + }), + ].map((event) => event!); +} + +export function defineTypeSafeInstrumentationAssertions(options: { + name: string; + runScenario: RunTypeSafeScenario; + snapshotName: string; + testFileUrl: string; +}): void { + const spanSnapshotPath = resolveFileSnapshotPath( + options.testFileUrl, + `${options.snapshotName}.span-tree.json`, + ); + + describe(options.name, () => { + let events: CapturedLogEvent[] = []; + + beforeAll(async () => { + await withScenarioHarness(async (harness) => { + await options.runScenario(harness); + events = harness.events(); + }); + }); + + test("captures native questions and every structured answer", () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan(events, "typesafe-mixed-operation"); + const span = findLatestChildSpan( + events, + "typesafe.systemOne", + operation?.span.id, + ); + + expect(root?.row.metadata).toMatchObject({ scenario: SCENARIO_NAME }); + expect(span?.span.type).toBe("llm"); + expect(span?.input).toMatchObject({ + state: { message: expect.any(String) }, + questions: { + category: { type: "choice" }, + urgency: { type: "score" }, + duplicate_charge: { type: "noul" }, + }, + }); + expect(span?.output).toMatchObject({ + category: { + type: "choice", + choice: expect.any(String), + confidence: expect.any(Number), + probabilities: expect.any(Object), + }, + urgency: { + type: "score", + score: expect.any(Number), + confidence: expect.any(Number), + legend: expect.any(Object), + probabilities: expect.any(Object), + }, + duplicate_charge: { type: "noul", noul: expect.any(Number) }, + }); + expect(span?.row.metadata).toMatchObject({ + model: expect.stringMatching(/^jev-/), + provider: "typesafe", + }); + expect(span?.metrics).toMatchObject({ + completion_tokens: expect.any(Number), + prompt_tokens: expect.any(Number), + tokens: expect.any(Number), + }); + }); + + test("captures output while preserving raw response access", () => { + const operation = findLatestSpan(events, "typesafe-raw-operation"); + const span = findLatestChildSpan( + events, + "typesafe.systemOne", + operation?.span.id, + ); + + expect(span?.input).toMatchObject({ + state: "The package arrived intact and on time.", + questions: { positive: { type: "noul" } }, + }); + expect(span?.output).toMatchObject({ + positive: { type: "noul", noul: expect.any(Number) }, + }); + expect(span?.row.metadata).toMatchObject({ + model: "jev-1.13.0", + provider: "typesafe", + }); + }); + + test("matches the span tree snapshot", async () => { + await matchSpanTreeSnapshot(spanTreeEvents(events), spanSnapshotPath); + }); + }); +} diff --git a/e2e/scenarios/typesafe-instrumentation/package.json b/e2e/scenarios/typesafe-instrumentation/package.json new file mode 100644 index 000000000..86f09750b --- /dev/null +++ b/e2e/scenarios/typesafe-instrumentation/package.json @@ -0,0 +1,18 @@ +{ + "name": "@braintrust/e2e-typesafe-instrumentation", + "private": true, + "braintrustScenario": { + "bump": { + "dependencies": { + "typesafe-sdk-v0-latest": { + "package": "@typesafe-ai/sdk", + "range": "0.6" + } + } + } + }, + "dependencies": { + "typesafe-sdk-v0": "npm:@typesafe-ai/sdk@0.6.0", + "typesafe-sdk-v0-latest": "npm:@typesafe-ai/sdk@0.6.0" + } +} diff --git a/e2e/scenarios/typesafe-instrumentation/pnpm-lock.yaml b/e2e/scenarios/typesafe-instrumentation/pnpm-lock.yaml new file mode 100644 index 000000000..0a4cad354 --- /dev/null +++ b/e2e/scenarios/typesafe-instrumentation/pnpm-lock.yaml @@ -0,0 +1,26 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + typesafe-sdk-v0: + specifier: npm:@typesafe-ai/sdk@0.6.0 + version: '@typesafe-ai/sdk@0.6.0' + typesafe-sdk-v0-latest: + specifier: npm:@typesafe-ai/sdk@0.6.0 + version: '@typesafe-ai/sdk@0.6.0' + +packages: + + '@typesafe-ai/sdk@0.6.0': + resolution: {integrity: sha512-IddX+Q0XM+VagOUZFeP7wZjaO4SHMdvnh2zEBdrZZnXedWI3BNK1lKhMx3ayrkFWvVLbVcUHJy6AVZlY+e6Jaw==} + engines: {node: '>=20'} + +snapshots: + + '@typesafe-ai/sdk@0.6.0': {} diff --git a/e2e/scenarios/typesafe-instrumentation/scenario.cjs-client.mjs b/e2e/scenarios/typesafe-instrumentation/scenario.cjs-client.mjs new file mode 100644 index 000000000..1ba916c2b --- /dev/null +++ b/e2e/scenarios/typesafe-instrumentation/scenario.cjs-client.mjs @@ -0,0 +1,9 @@ +import { createRequire } from "node:module"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoTypeSafeInstrumentation } from "./scenario.impl.mjs"; + +const require = createRequire(import.meta.url); +const packageName = process.env.TYPESAFE_PACKAGE_NAME ?? "typesafe-sdk-v0"; +const typesafe = require(packageName); + +runMain(async () => runAutoTypeSafeInstrumentation(typesafe)); diff --git a/e2e/scenarios/typesafe-instrumentation/scenario.cjs-client.ts b/e2e/scenarios/typesafe-instrumentation/scenario.cjs-client.ts new file mode 100644 index 000000000..c4fb20ed4 --- /dev/null +++ b/e2e/scenarios/typesafe-instrumentation/scenario.cjs-client.ts @@ -0,0 +1,10 @@ +import { createRequire } from "node:module"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runWrappedTypeSafeInstrumentation } from "./scenario.impl.mjs"; + +const require = createRequire(import.meta.url); +const packageName = process.env.TYPESAFE_PACKAGE_NAME ?? "typesafe-sdk-v0"; + +runMain(async () => { + await runWrappedTypeSafeInstrumentation(require(packageName)); +}); diff --git a/e2e/scenarios/typesafe-instrumentation/scenario.impl.mjs b/e2e/scenarios/typesafe-instrumentation/scenario.impl.mjs new file mode 100644 index 000000000..d5eee5229 --- /dev/null +++ b/e2e/scenarios/typesafe-instrumentation/scenario.impl.mjs @@ -0,0 +1,90 @@ +import { wrapTypeSafe } from "braintrust"; +import { + runOperation, + runTracedScenario, +} from "../../helpers/provider-runtime.mjs"; + +export const ROOT_NAME = "typesafe-instrumentation-root"; +export const SCENARIO_NAME = "typesafe-instrumentation"; + +export async function runTypeSafeInstrumentationScenario( + typesafe, + { decorateClient } = {}, +) { + if (!process.env.TYPESAFE_API_KEY) { + throw new Error("Expected TYPESAFE_API_KEY to be set for e2e"); + } + + const baseClient = new typesafe.TypeSafeClient(); + const client = decorateClient ? decorateClient(baseClient) : baseClient; + + await runTracedScenario({ + callback: async () => { + await runOperation("typesafe-mixed-operation", "mixed", async () => { + const promise = client.systemOne({ + state: { + message: + "I was charged twice. Please refund the duplicate charge today.", + }, + questions: { + category: typesafe.choice("Which team should handle this?", { + billing: null, + technical: null, + other: null, + }), + urgency: typesafe.score("How urgent is this request?", [ + "routine", + "soon", + "urgent", + ]), + duplicate_charge: typesafe.noul({ + question: "Does the customer report a duplicate charge?", + }), + }, + }); + const mapped = promise.map((result) => result.answers.category.choice); + const [{ data, requestId }, category] = await Promise.all([ + promise.withResponse(), + mapped, + ]); + if (!requestId || category !== data.answers.category.choice) { + throw new Error("TypeSafe APIPromise helpers were not preserved"); + } + }); + + await runOperation("typesafe-raw-operation", "raw", async () => { + const response = await client + .systemOne({ + model: "jev-1.13.0", + state: "The package arrived intact and on time.", + questions: { + positive: typesafe.noul("Is this feedback positive?"), + }, + }) + .asResponse(); + if (response.bodyUsed) { + throw new Error( + "Instrumentation consumed the caller's response body", + ); + } + const data = await response.json(); + if (data.answers?.positive?.type !== "noul") { + throw new Error("Unexpected TypeSafe raw response"); + } + }); + }, + metadata: { scenario: SCENARIO_NAME }, + projectNameBase: "tmp-luca-e2e-typesafe-instrumentation", + rootName: ROOT_NAME, + }); +} + +export async function runWrappedTypeSafeInstrumentation(typesafe) { + await runTypeSafeInstrumentationScenario(typesafe, { + decorateClient: wrapTypeSafe, + }); +} + +export async function runAutoTypeSafeInstrumentation(typesafe) { + await runTypeSafeInstrumentationScenario(typesafe); +} diff --git a/e2e/scenarios/typesafe-instrumentation/scenario.mjs b/e2e/scenarios/typesafe-instrumentation/scenario.mjs new file mode 100644 index 000000000..d0d861533 --- /dev/null +++ b/e2e/scenarios/typesafe-instrumentation/scenario.mjs @@ -0,0 +1,7 @@ +const packageName = + process.env.TYPESAFE_PACKAGE_NAME ?? "typesafe-sdk-v0-latest"; +const typesafe = await import(packageName); +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoTypeSafeInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runAutoTypeSafeInstrumentation(typesafe)); diff --git a/e2e/scenarios/typesafe-instrumentation/scenario.test.ts b/e2e/scenarios/typesafe-instrumentation/scenario.test.ts new file mode 100644 index 000000000..838b56b1e --- /dev/null +++ b/e2e/scenarios/typesafe-instrumentation/scenario.test.ts @@ -0,0 +1,76 @@ +import { describe } from "vitest"; +import { + prepareScenarioDir, + readInstalledPackageVersion, + resolveScenarioDir, +} from "../../helpers/scenario-harness"; +import { defineTypeSafeInstrumentationAssertions } from "./assertions"; + +const originalScenarioDir = resolveScenarioDir(import.meta.url); +const scenarioDir = await prepareScenarioDir({ + scenarioDir: originalScenarioDir, +}); + +const scenarios = await Promise.all( + [ + { + autoEntry: "scenario.cjs-client.mjs", + dependencyName: "typesafe-sdk-v0", + snapshotName: "typesafe-v0", + wrapperEntry: "scenario.cjs-client.ts", + }, + { + autoEntry: "scenario.mjs", + dependencyName: "typesafe-sdk-v0-latest", + snapshotName: "typesafe-v0-latest", + wrapperEntry: "scenario.ts", + }, + ].map(async (scenario) => ({ + ...scenario, + version: await readInstalledPackageVersion( + scenarioDir, + scenario.dependencyName, + ), + })), +); + +describe.concurrent("variants", () => { + for (const scenario of scenarios) { + describe.sequential(`typesafe sdk ${scenario.version}`, () => { + const env = { TYPESAFE_PACKAGE_NAME: scenario.dependencyName }; + const runContext = { + originalScenarioDir, + variantKey: scenario.snapshotName, + }; + + defineTypeSafeInstrumentationAssertions({ + name: "wrapped instrumentation", + runScenario: async ({ runScenarioDir }) => { + await runScenarioDir({ + entry: scenario.wrapperEntry, + env, + runContext, + scenarioDir, + }); + }, + snapshotName: `${scenario.snapshotName}-wrapped`, + testFileUrl: import.meta.url, + }); + + defineTypeSafeInstrumentationAssertions({ + name: "auto-hook instrumentation", + runScenario: async ({ runNodeScenarioDir }) => { + await runNodeScenarioDir({ + entry: scenario.autoEntry, + env, + nodeArgs: ["--import", "braintrust/hook.mjs"], + runContext, + scenarioDir, + }); + }, + snapshotName: scenario.snapshotName, + testFileUrl: import.meta.url, + }); + }); + } +}); diff --git a/e2e/scenarios/typesafe-instrumentation/scenario.ts b/e2e/scenarios/typesafe-instrumentation/scenario.ts new file mode 100644 index 000000000..dffbe54a6 --- /dev/null +++ b/e2e/scenarios/typesafe-instrumentation/scenario.ts @@ -0,0 +1,9 @@ +const packageName = + process.env.TYPESAFE_PACKAGE_NAME ?? "typesafe-sdk-v0-latest"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runWrappedTypeSafeInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => { + const typesafe = await import(packageName); + await runWrappedTypeSafeInstrumentation(typesafe); +}); diff --git a/e2e/scripts/run-e2e-bump-docker.mjs b/e2e/scripts/run-e2e-bump-docker.mjs index c36f56f35..93b547b3d 100644 --- a/e2e/scripts/run-e2e-bump-docker.mjs +++ b/e2e/scripts/run-e2e-bump-docker.mjs @@ -64,6 +64,8 @@ const ALLOWED_ENV_KEYS = [ "OPENAI_CODEX_E2E_MODEL", "OPENROUTER_API_KEY", "OPENROUTER_BASE_URL", + "TYPESAFE_API_KEY", + "TYPESAFE_BASE_URL", ]; function loadRepoEnv() { diff --git a/js/src/auto-instrumentations/configs/all.ts b/js/src/auto-instrumentations/configs/all.ts index bd2d45057..deb7aad8f 100644 --- a/js/src/auto-instrumentations/configs/all.ts +++ b/js/src/auto-instrumentations/configs/all.ts @@ -36,6 +36,7 @@ import { piCodingAgentConfigs } from "./pi-coding-agent"; import { strandsAgentSDKConfigs } from "./strands-agent-sdk"; import { elevenLabsConfigs } from "./elevenlabs"; import { voyageAIConfigs } from "./voyageai"; +import { typeSafeConfigs } from "./typesafe"; interface InstrumentationConfigGroup { integrations: readonly (keyof InstrumentationIntegrationsConfig)[]; @@ -135,6 +136,10 @@ const defaultInstrumentationConfigGroups: readonly InstrumentationConfigGroup[] integrations: ["voyageai"], configs: voyageAIConfigs, }, + { + integrations: ["typesafe"], + configs: typeSafeConfigs, + }, // Note: `@mastra/core` is not listed here because its instrumentation // doesn't go through the AST `code-transformer` matcher — Mastra's // content-hashed chunks make `filePath`-based matching too brittle. diff --git a/js/src/auto-instrumentations/configs/typesafe.ts b/js/src/auto-instrumentations/configs/typesafe.ts new file mode 100644 index 000000000..2d64e4fa2 --- /dev/null +++ b/js/src/auto-instrumentations/configs/typesafe.ts @@ -0,0 +1,19 @@ +import type { InstrumentationConfig } from "../orchestrion-js"; +import { typeSafeChannels } from "../../instrumentation/plugins/typesafe-channels"; + +export const typeSafeConfigs: InstrumentationConfig[] = [ + "dist/index.mjs", + "dist/index.cjs", +].map((filePath) => ({ + channelName: typeSafeChannels.systemOne.channelName, + module: { + name: "@typesafe-ai/sdk", + versionRange: ">=0.6.0 <1.0.0", + filePath, + }, + functionQuery: { + className: "TypeSafeClient", + methodName: "systemOne", + kind: "Async" as const, + }, +})); diff --git a/js/src/exports.ts b/js/src/exports.ts index 0d4d20b2f..683b3f868 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -238,6 +238,7 @@ export { wrapLangGraphSDK } from "./wrappers/langgraph-sdk"; export { wrapOllama } from "./wrappers/ollama"; export { wrapCohere } from "./wrappers/cohere"; export { wrapVoyageAI } from "./wrappers/voyageai"; +export { wrapTypeSafe } from "./wrappers/typesafe"; export { wrapGroq } from "./wrappers/groq"; export { wrapBedrockRuntime } from "./wrappers/bedrock-runtime"; export { wrapCopilotClient } from "./wrappers/github-copilot"; diff --git a/js/src/instrumentation/braintrust-plugin.test.ts b/js/src/instrumentation/braintrust-plugin.test.ts index 6dfacf5cb..40e12fbfa 100644 --- a/js/src/instrumentation/braintrust-plugin.test.ts +++ b/js/src/instrumentation/braintrust-plugin.test.ts @@ -23,6 +23,7 @@ import { LangSmithPlugin } from "./plugins/langsmith-plugin"; import { PiCodingAgentPlugin } from "./plugins/pi-coding-agent-plugin"; import { StrandsAgentSDKPlugin } from "./plugins/strands-agent-sdk-plugin"; import { VoyageAIPlugin } from "./plugins/voyageai-plugin"; +import { TypeSafePlugin } from "./plugins/typesafe-plugin"; import { CloudflareAIChatPlugin } from "./plugins/cloudflare-ai-chat-plugin"; import { CloudflareAgentsPlugin } from "./plugins/cloudflare-agents-plugin"; @@ -138,6 +139,10 @@ vi.mock("./plugins/voyageai-plugin", () => ({ VoyageAIPlugin: createPluginClassMock(), })); +vi.mock("./plugins/typesafe-plugin", () => ({ + TypeSafePlugin: createPluginClassMock(), +})); + vi.mock("./plugins/cloudflare-ai-chat-plugin", () => ({ CloudflareAIChatPlugin: createPluginClassMock(), })); @@ -286,6 +291,15 @@ describe("BraintrustPlugin", () => { expect(mockInstance.enable).toHaveBeenCalledTimes(1); }); + it("should create and enable TypeSafe plugin by default", () => { + const plugin = new BraintrustPlugin(); + plugin.enable(); + + expect(TypeSafePlugin).toHaveBeenCalledTimes(1); + const mockInstance = vi.mocked(TypeSafePlugin).mock.results[0].value; + expect(mockInstance.enable).toHaveBeenCalledTimes(1); + }); + it("should create and enable Groq plugin by default", () => { const plugin = new BraintrustPlugin(); plugin.enable(); @@ -1198,6 +1212,15 @@ describe("BraintrustPlugin", () => { expect(VoyageAIPlugin).not.toHaveBeenCalled(); }); + + it("should not create TypeSafe plugin when typesafe: false", () => { + const plugin = new BraintrustPlugin({ + integrations: { typesafe: false }, + }); + plugin.enable(); + + expect(TypeSafePlugin).not.toHaveBeenCalled(); + }); }); }); diff --git a/js/src/instrumentation/braintrust-plugin.ts b/js/src/instrumentation/braintrust-plugin.ts index 942e7fe7c..93015d351 100644 --- a/js/src/instrumentation/braintrust-plugin.ts +++ b/js/src/instrumentation/braintrust-plugin.ts @@ -29,6 +29,7 @@ import { PiCodingAgentPlugin } from "./plugins/pi-coding-agent-plugin"; import { StrandsAgentSDKPlugin } from "./plugins/strands-agent-sdk-plugin"; import { ElevenLabsPlugin } from "./plugins/elevenlabs-plugin"; import { VoyageAIPlugin } from "./plugins/voyageai-plugin"; +import { TypeSafePlugin } from "./plugins/typesafe-plugin"; import { CloudflareAIChatPlugin } from "./plugins/cloudflare-ai-chat-plugin"; import { CloudflareAgentsPlugin } from "./plugins/cloudflare-agents-plugin"; import type { InstrumentationIntegrationsConfig } from "./config"; @@ -89,6 +90,7 @@ export class BraintrustPlugin extends BasePlugin { private strandsAgentSDKPlugin: StrandsAgentSDKPlugin | null = null; private elevenLabsPlugin: ElevenLabsPlugin | null = null; private voyageAIPlugin: VoyageAIPlugin | null = null; + private typeSafePlugin: TypeSafePlugin | null = null; private cloudflareAIChatPlugin: CloudflareAIChatPlugin | null = null; private cloudflareAgentsPlugin: CloudflareAgentsPlugin | null = null; @@ -211,6 +213,11 @@ export class BraintrustPlugin extends BasePlugin { this.voyageAIPlugin.enable(); } + if (integrations.typesafe !== false) { + this.typeSafePlugin = new TypeSafePlugin(); + this.typeSafePlugin.enable(); + } + if (integrations.groq !== false) { this.groqPlugin = new GroqPlugin(); this.groqPlugin.enable(); @@ -384,6 +391,11 @@ export class BraintrustPlugin extends BasePlugin { this.voyageAIPlugin = null; } + if (this.typeSafePlugin) { + this.typeSafePlugin.disable(); + this.typeSafePlugin = null; + } + if (this.groqPlugin) { this.groqPlugin.disable(); this.groqPlugin = null; diff --git a/js/src/instrumentation/config.ts b/js/src/instrumentation/config.ts index 784ca7b2b..2f969dd58 100644 --- a/js/src/instrumentation/config.ts +++ b/js/src/instrumentation/config.ts @@ -36,6 +36,7 @@ export interface InstrumentationIntegrationsConfig { langgraphSDK?: boolean; langsmith?: boolean; voyageai?: boolean; + typesafe?: boolean; elevenlabs?: boolean; } @@ -130,6 +131,9 @@ const envIntegrationAliases: Record< voyage: "voyageai", "voyage-ai": "voyageai", voyageai: "voyageai", + typesafe: "typesafe", + "typesafe-ai": "typesafe", + "@typesafe-ai/sdk": "typesafe", elevenlabs: "elevenlabs", "@elevenlabs/elevenlabs-js": "elevenlabs", }; @@ -173,6 +177,7 @@ export function getDefaultInstrumentationIntegrations(): Record< langgraphSDK: true, langsmith: true, voyageai: true, + typesafe: true, elevenlabs: true, piCodingAgent: true, strandsAgentSDK: true, diff --git a/js/src/instrumentation/plugins/typesafe-channels.ts b/js/src/instrumentation/plugins/typesafe-channels.ts new file mode 100644 index 000000000..48848c02b --- /dev/null +++ b/js/src/instrumentation/plugins/typesafe-channels.ts @@ -0,0 +1,24 @@ +import { INSTRUMENTATION_NAMES } from "../../span-origin"; +import type { + TypeSafeAPIPromise, + TypeSafeSystemOneRequest, + TypeSafeSystemOneResult, +} from "../../vendor-sdk-types/typesafe"; +import { channel, defineChannels } from "../core/channel-definitions"; + +export const typeSafeChannels = defineChannels( + "@typesafe-ai/sdk", + { + systemOne: channel< + [TypeSafeSystemOneRequest, options?: unknown], + TypeSafeSystemOneResult + >({ + channelName: "systemOne", + kind: "async", + }), + }, + { instrumentationName: INSTRUMENTATION_NAMES.TYPESAFE }, +); + +export type TypeSafeSystemOnePromise = + TypeSafeAPIPromise; diff --git a/js/src/instrumentation/plugins/typesafe-plugin.ts b/js/src/instrumentation/plugins/typesafe-plugin.ts new file mode 100644 index 000000000..7bc8d6a47 --- /dev/null +++ b/js/src/instrumentation/plugins/typesafe-plugin.ts @@ -0,0 +1,271 @@ +import { SpanTypeAttribute, isObject } from "../../../util/index"; +import { debugLogger } from "../../debug-logger"; +import { startSpan, withCurrent } from "../../logger"; +import type { Span } from "../../logger"; +import { + INSTRUMENTATION_NAMES, + withSpanInstrumentationName, +} from "../../span-origin"; +import type { + TypeSafeAPIPromise, + TypeSafeSystemOneRequest, + TypeSafeSystemOneResult, +} from "../../vendor-sdk-types/typesafe"; +import { + isAutoInstrumentationSuppressed, + runWithAutoInstrumentationSuppressed, +} from "../auto-instrumentation-suppression"; +import { BasePlugin } from "../core"; +import { unsubscribeAll } from "../core/channel-tracing"; +import { typeSafeChannels } from "./typesafe-channels"; + +export class TypeSafePlugin extends BasePlugin { + protected onEnable(): void { + this.unsubscribers.push( + typeSafeChannels.systemOne.intercept((target, thisArg, args) => + interceptSystemOne(target, thisArg, args), + ), + ); + } + + protected onDisable(): void { + this.unsubscribers = unsubscribeAll(this.unsubscribers); + } +} + +function interceptSystemOne( + target: ( + this: unknown, + request: TypeSafeSystemOneRequest, + options?: unknown, + ) => PromiseLike, + thisArg: unknown, + args: [TypeSafeSystemOneRequest, options?: unknown], +): PromiseLike { + const invokeTarget = () => Reflect.apply(target, thisArg, args); + if (isAutoInstrumentationSuppressed()) { + return invokeTarget(); + } + + let span: Span; + try { + const request = args[0]; + const model = + typeof request?.model === "string" + ? request.model + : isObject(thisArg) && typeof thisArg.defaultModel === "string" + ? thisArg.defaultModel + : undefined; + span = startSpan( + withSpanInstrumentationName( + { + event: { + input: { + state: request?.state, + questions: request?.questions, + }, + metadata: { + ...(model ? { model } : {}), + provider: "typesafe", + }, + }, + name: "typesafe.systemOne", + spanAttributes: { type: SpanTypeAttribute.LLM }, + }, + INSTRUMENTATION_NAMES.TYPESAFE, + ), + ); + } catch (error) { + debugLogger.error("Error starting span for typesafe.systemOne:", error); + return invokeTarget(); + } + + let result: PromiseLike; + try { + result = withCurrent(span, () => + runWithAutoInstrumentationSuppressed(invokeTarget), + ); + } catch (error) { + finishTypeSafeSpan(span, () => span.log({ error })); + throw error; + } + + if (!isTypeSafeAPIPromise(result)) { + const finished = Promise.resolve(result).then( + (value) => finishSuccessfulSpan(span, value), + (error) => finishTypeSafeSpan(span, () => span.log({ error })), + ); + void finished; + return result; + } + + const originalAsResponse = result.asResponse.bind(result); + const captureFinished = originalAsResponse().then( + async (response) => { + try { + const value = await response.clone().json(); + if (!isObject(value)) { + throw new TypeError("Expected TypeSafe to return a JSON object"); + } + finishSuccessfulSpan(span, value); + } catch (error) { + debugLogger.error( + "Error reading response for typesafe.systemOne:", + error, + ); + finishTypeSafeSpan(span, () => {}); + } + }, + (error) => finishTypeSafeSpan(span, () => span.log({ error })), + ); + preserveTypeSafePromise(result, captureFinished); + return result; +} + +function isTypeSafeAPIPromise( + value: PromiseLike, +): value is TypeSafeAPIPromise { + return ( + (typeof value === "object" || typeof value === "function") && + value !== null && + typeof Reflect.get(value, "asResponse") === "function" && + typeof Reflect.get(value, "withResponse") === "function" && + typeof Reflect.get(value, "map") === "function" + ); +} + +function preserveTypeSafePromise( + apiPromise: TypeSafeAPIPromise, + captureFinished: Promise, +): void { + try { + const originalThen = apiPromise.then.bind(apiPromise); + const originalAsResponse = apiPromise.asResponse.bind(apiPromise); + const originalWithResponse = apiPromise.withResponse.bind(apiPromise); + const originalMap = apiPromise.map.bind(apiPromise); + + const waitForCapture = async (value: V): Promise => { + await captureFinished; + return value; + }; + const then = ( + onfulfilled?: ((value: T) => TResult1 | PromiseLike) | null, + onrejected?: + | ((reason: unknown) => TResult2 | PromiseLike) + | null, + ): Promise => + originalThen( + async (value) => { + await captureFinished; + if (onfulfilled) { + return onfulfilled(value); + } + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions -- Promise.then returns the original value when no callback is supplied. + return value as unknown as TResult1; + }, + async (error) => { + await captureFinished; + if (onrejected) { + return onrejected(error); + } + throw error; + }, + ); + + Object.defineProperties(apiPromise, { + asResponse: { + configurable: true, + value: () => originalAsResponse().then(waitForCapture), + }, + catch: { + configurable: true, + value: ( + onrejected?: + | ((reason: unknown) => TResult | PromiseLike) + | null, + ) => then(undefined, onrejected), + }, + finally: { + configurable: true, + value: (onfinally?: (() => void) | null) => + then( + async (value) => { + await onfinally?.(); + return value; + }, + async (error) => { + await onfinally?.(); + throw error; + }, + ), + }, + map: { + configurable: true, + value: (fn: (data: T) => U) => { + const mapped = originalMap(fn); + preserveTypeSafePromise(mapped, captureFinished); + return mapped; + }, + }, + then: { configurable: true, value: then }, + withResponse: { + configurable: true, + value: () => originalWithResponse().then(waitForCapture), + }, + }); + } catch (error) { + debugLogger.error("Error preserving TypeSafe APIPromise helpers:", error); + } +} + +function finishSuccessfulSpan( + span: Span, + result: TypeSafeSystemOneResult, +): void { + finishTypeSafeSpan(span, () => { + const metrics = extractMetrics(result); + span.log({ + output: isObject(result) ? result.answers : undefined, + ...(isObject(result) && typeof result.model === "string" + ? { metadata: { model: result.model } } + : {}), + ...(Object.keys(metrics).length > 0 ? { metrics } : {}), + }); + }); +} + +function extractMetrics( + result: TypeSafeSystemOneResult, +): Record { + const usage = isObject(result) && isObject(result.usage) ? result.usage : {}; + const promptTokens = validTokenCount(usage.input_tokens); + const completionTokens = validTokenCount(usage.output_tokens); + return { + ...(promptTokens !== undefined ? { prompt_tokens: promptTokens } : {}), + ...(completionTokens !== undefined + ? { completion_tokens: completionTokens } + : {}), + ...(promptTokens !== undefined && completionTokens !== undefined + ? { tokens: promptTokens + completionTokens } + : {}), + }; +} + +function validTokenCount(value: unknown): number | undefined { + return typeof value === "number" && Number.isInteger(value) && value >= 0 + ? value + : undefined; +} + +function finishTypeSafeSpan(span: Span, log: () => void): void { + try { + log(); + } catch (error) { + debugLogger.error("Error logging span for typesafe.systemOne:", error); + } + try { + span.end(); + } catch (error) { + debugLogger.error("Error ending span for typesafe.systemOne:", error); + } +} diff --git a/js/src/span-origin.ts b/js/src/span-origin.ts index ffc9cd037..0c1afb303 100644 --- a/js/src/span-origin.ts +++ b/js/src/span-origin.ts @@ -41,6 +41,7 @@ export const INSTRUMENTATION_NAMES = { OPENROUTER_AGENT: "openrouter-agent", PI_CODING_AGENT: "pi-coding-agent", STRANDS_AGENT_SDK: "strands-agent-sdk", + TYPESAFE: "typesafe", VOYAGEAI: "voyageai", ELEVENLABS: "elevenlabs", } as const; diff --git a/js/src/vendor-sdk-types/typesafe.ts b/js/src/vendor-sdk-types/typesafe.ts new file mode 100644 index 000000000..83ee52919 --- /dev/null +++ b/js/src/vendor-sdk-types/typesafe.ts @@ -0,0 +1,44 @@ +export type TypeSafeSystemOneRequest = { + state: unknown; + questions: Record; + model?: string; + [key: string]: unknown; +}; + +export type TypeSafeUsage = { + input_tokens?: number; + output_tokens?: number; + [key: string]: unknown; +}; + +export type TypeSafeSystemOneResult = { + model?: string; + answers?: Record; + usage?: TypeSafeUsage; + [key: string]: unknown; +}; + +export type TypeSafeWithResponse = { + data: T; + response: Response; + requestId?: string; +}; + +export type TypeSafeAPIPromise = Promise & { + asResponse(): Promise; + withResponse(): Promise>; + map(fn: (data: T) => U): TypeSafeAPIPromise; + catch( + onrejected?: ((reason: unknown) => TResult | PromiseLike) | null, + ): Promise; + finally(onfinally?: (() => void) | null): Promise; +}; + +export type TypeSafeClient = { + defaultModel?: string; + systemOne( + request: TypeSafeSystemOneRequest, + options?: unknown, + ): TypeSafeAPIPromise; + [key: string]: unknown; +}; diff --git a/js/src/wrappers/typesafe.test.ts b/js/src/wrappers/typesafe.test.ts new file mode 100644 index 000000000..4ed46740c --- /dev/null +++ b/js/src/wrappers/typesafe.test.ts @@ -0,0 +1,235 @@ +import { + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; +import { _exportsForTestingOnly, initLogger } from "../logger"; +import { configureNode } from "../node/config"; +import type { + TypeSafeAPIPromise, + TypeSafeWithResponse, +} from "../vendor-sdk-types/typesafe"; +import { wrapTypeSafe } from "./typesafe"; + +try { + configureNode(); +} catch { + // Best-effort initialization for test environments. +} + +class MockAPIPromise implements TypeSafeAPIPromise { + readonly [Symbol.toStringTag] = "Promise"; + private parsed: Promise | undefined; + + constructor( + private readonly responsePromise: Promise, + private readonly parse: (response: Response) => Promise, + ) {} + + private data(): Promise { + this.parsed ??= this.responsePromise.then(this.parse); + return this.parsed; + } + + then( + onfulfilled?: ((value: T) => TResult1 | PromiseLike) | null, + onrejected?: ((reason: unknown) => TResult2 | PromiseLike) | null, + ): Promise { + return this.data().then(onfulfilled, onrejected); + } + + catch( + onrejected?: ((reason: unknown) => TResult | PromiseLike) | null, + ): Promise { + return this.data().catch(onrejected); + } + + finally(onfinally?: (() => void) | null): Promise { + return this.data().finally(onfinally); + } + + asResponse(): Promise { + return this.responsePromise; + } + + async withResponse(): Promise> { + const [data, response] = await Promise.all([ + this.data(), + this.responsePromise, + ]); + return { data, response, requestId: "req_test" }; + } + + map(fn: (data: T) => U): MockAPIPromise { + return new MockAPIPromise(this.responsePromise, async () => + fn(await this.data()), + ); + } +} + +function responsePromise(body: unknown): MockAPIPromise { + const response = Promise.resolve( + new Response(JSON.stringify(body), { + headers: { "content-type": "application/json" }, + }), + ); + return new MockAPIPromise(response, async (value) => value.json()); +} + +describe("TypeSafe wrapper", () => { + let backgroundLogger: ReturnType< + typeof _exportsForTestingOnly.useTestBackgroundLogger + >; + + beforeAll(async () => { + await _exportsForTestingOnly.simulateLoginForTests(); + }); + + beforeEach(() => { + backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); + initLogger({ + projectName: "typesafe.test.ts", + projectId: "test-project-id", + }); + }); + + afterEach(() => { + _exportsForTestingOnly.clearTestBackgroundLogger(); + vi.restoreAllMocks(); + }); + + it("returns unsupported clients unchanged", () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + const unsupported = { models: {} }; + + expect(wrapTypeSafe(unsupported)).toBe(unsupported); + expect(warnSpy).toHaveBeenCalledWith( + "Unsupported TypeSafe AI library. Not wrapping.", + ); + }); + + it("captures native questions, answers, model, and token usage", async () => { + const result = { + answers: { + category: { + choice: "billing", + confidence: 0.9, + probabilities: { billing: 0.95, technical: 0.05 }, + type: "choice", + }, + urgent: { noul: 0.8, type: "noul" }, + }, + model: "jev-1.13.0", + usage: { input_tokens: 20, output_tokens: 5 }, + }; + const original = responsePromise(result); + const client = wrapTypeSafe({ + defaultModel: "jev-latest", + systemOne: vi.fn(() => original), + } as any) as any; + + const returned = client.systemOne({ + state: { message: "Charged twice" }, + questions: { + category: { + criteria: { billing: null, technical: null }, + instructions: "Which team?", + type: "choice", + }, + urgent: { instructions: "Is this urgent?", type: "noul" }, + }, + }); + + expect(returned).toBe(original); + await expect(returned).resolves.toEqual(result); + + const spans = await backgroundLogger.drain(); + const span = spans.find( + (candidate: any) => + candidate.span_attributes?.name === "typesafe.systemOne", + ) as Record | undefined; + expect(span).toMatchObject({ + input: { + questions: { + category: { type: "choice" }, + urgent: { type: "noul" }, + }, + state: { message: "Charged twice" }, + }, + metadata: { model: "jev-1.13.0", provider: "typesafe" }, + metrics: { completion_tokens: 5, prompt_tokens: 20, tokens: 25 }, + output: result.answers, + span_attributes: { name: "typesafe.systemOne", type: "llm" }, + }); + }); + + it("preserves APIPromise helpers and the raw response body", async () => { + const result = { + answers: { valid: { noul: 1, type: "noul" } }, + model: "jev-1.13.0", + usage: { input_tokens: 4, output_tokens: 1 }, + }; + const original = responsePromise(result); + const client = wrapTypeSafe({ + systemOne: vi.fn(() => original), + } as any) as any; + + const promise = client.systemOne({ state: "hello", questions: {} }); + expect(promise).toBe(original); + expect(typeof promise.asResponse).toBe("function"); + expect(typeof promise.withResponse).toBe("function"); + expect(typeof promise.map).toBe("function"); + + const raw = await promise.asResponse(); + expect(raw.bodyUsed).toBe(false); + await expect(raw.json()).resolves.toEqual(result); + + const mappedPromise = responsePromise(result); + const mappedClient = wrapTypeSafe({ + systemOne: vi.fn(() => mappedPromise), + } as any) as any; + const mapped = mappedClient + .systemOne({ state: "hello", questions: {} }) + .map((value: any) => value.model); + await expect(mapped).resolves.toBe("jev-1.13.0"); + }); + + it("preserves synchronous and asynchronous provider errors", async () => { + const syncError = new Error("invalid questions"); + const syncClient = wrapTypeSafe({ + systemOne: vi.fn(() => { + throw syncError; + }), + } as any) as any; + + expect(() => + syncClient.systemOne({ state: "hello", questions: {} }), + ).toThrow(syncError); + + const asyncError = new Error("request failed"); + const rejected = new MockAPIPromise( + Promise.reject(asyncError), + async () => undefined, + ); + const asyncClient = wrapTypeSafe({ + systemOne: vi.fn(() => rejected), + } as any) as any; + await expect( + asyncClient.systemOne({ state: "hello", questions: {} }), + ).rejects.toBe(asyncError); + + const spans = await backgroundLogger.drain(); + const errors = spans + .filter( + (candidate: any) => + candidate.span_attributes?.name === "typesafe.systemOne", + ) + .map((span: any) => span.error); + expect(errors).toHaveLength(2); + expect(errors.every((error) => error !== undefined)).toBe(true); + }); +}); diff --git a/js/src/wrappers/typesafe.ts b/js/src/wrappers/typesafe.ts new file mode 100644 index 000000000..fc6055322 --- /dev/null +++ b/js/src/wrappers/typesafe.ts @@ -0,0 +1,48 @@ +import { typeSafeChannels } from "../instrumentation/plugins/typesafe-channels"; +import type { + TypeSafeClient, + TypeSafeSystemOneRequest, +} from "../vendor-sdk-types/typesafe"; + +/** Wrap a TypeSafe client so systemOne calls pass through Braintrust tracing. */ +export function wrapTypeSafe(client: T): T; +export function wrapTypeSafe(client: unknown): unknown { + if (!isSupportedTypeSafeClient(client)) { + // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. + console.warn("Unsupported TypeSafe AI library. Not wrapping."); + return client; + } + + const cached = typeSafeProxyCache.get(client); + if (cached) { + return cached; + } + + const proxy = new Proxy(client, { + get(target, prop, receiver) { + if (prop !== "systemOne") { + return Reflect.get(target, prop, receiver); + } + + return (request: TypeSafeSystemOneRequest, options?: unknown) => + typeSafeChannels.systemOne.invoke( + target.systemOne, + target, + [request, options], + {}, + ); + }, + }); + typeSafeProxyCache.set(client, proxy); + return proxy; +} + +const typeSafeProxyCache = new WeakMap(); + +function isSupportedTypeSafeClient(value: unknown): value is TypeSafeClient { + return ( + (typeof value === "object" || typeof value === "function") && + value !== null && + typeof Reflect.get(value, "systemOne") === "function" + ); +} From b8fc79f712bdb67ee6ffb18e5e052f0e233f3866 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Thu, 17 Sep 2026 11:53:51 +0000 Subject: [PATCH 2/4] Update PR #2493 --- .changeset/add-typesafe-instrumentation.md | 5 +++ .../plugins/typesafe-channels.ts | 4 --- js/src/span-cache.test.ts | 31 +++++++++++++++++-- js/src/span-cache.ts | 28 +++++++++++++++-- 4 files changed, 59 insertions(+), 9 deletions(-) create mode 100644 .changeset/add-typesafe-instrumentation.md diff --git a/.changeset/add-typesafe-instrumentation.md b/.changeset/add-typesafe-instrumentation.md new file mode 100644 index 000000000..d17219245 --- /dev/null +++ b/.changeset/add-typesafe-instrumentation.md @@ -0,0 +1,5 @@ +--- +"braintrust": minor +--- + +feat: Add instrumentation for `@typesafe-ai/sdk` diff --git a/js/src/instrumentation/plugins/typesafe-channels.ts b/js/src/instrumentation/plugins/typesafe-channels.ts index 48848c02b..5c54fa41b 100644 --- a/js/src/instrumentation/plugins/typesafe-channels.ts +++ b/js/src/instrumentation/plugins/typesafe-channels.ts @@ -1,6 +1,5 @@ import { INSTRUMENTATION_NAMES } from "../../span-origin"; import type { - TypeSafeAPIPromise, TypeSafeSystemOneRequest, TypeSafeSystemOneResult, } from "../../vendor-sdk-types/typesafe"; @@ -19,6 +18,3 @@ export const typeSafeChannels = defineChannels( }, { instrumentationName: INSTRUMENTATION_NAMES.TYPESAFE }, ); - -export type TypeSafeSystemOnePromise = - TypeSafeAPIPromise; diff --git a/js/src/span-cache.test.ts b/js/src/span-cache.test.ts index 8479da30f..37760b00f 100644 --- a/js/src/span-cache.test.ts +++ b/js/src/span-cache.test.ts @@ -5,7 +5,9 @@ import { beforeEach, afterEach, beforeAll, + vi, } from "vitest"; +import iso from "./isomorph"; import { SpanCache } from "./span-cache"; import { configureNode } from "./node/config"; @@ -21,8 +23,10 @@ describe("SpanCache (disk-based)", () => { cache.start(); // Start for testing (cache is disabled by default) }); - afterEach(() => { - // Clean up temp file after each test + afterEach(async () => { + // Finish pending file operations before closing and removing the cache. + await cache.waitForPendingWrites(); + cache.stop(); cache.dispose(); }); @@ -149,6 +153,29 @@ describe("SpanCache (disk-based)", () => { }); describe("dispose", () => { + test("closes a file handle when initialization finishes after disposal", async () => { + const close = vi.fn(async () => {}); + let resolveOpen: (fileHandle: { + close: () => Promise; + }) => void = () => {}; + const openPromise = new Promise<{ close: () => Promise }>( + (resolve) => { + resolveOpen = resolve; + }, + ); + const openFile = vi.spyOn(iso, "openFile").mockReturnValue(openPromise); + const racingCache = new SpanCache(); + + racingCache.start(); + racingCache.queueWrite("root-1", "span-1", { span_id: "span-1" }); + racingCache.stop(); + racingCache.dispose(); + resolveOpen({ close }); + + await vi.waitFor(() => expect(close).toHaveBeenCalledOnce()); + openFile.mockRestore(); + }); + test("should clean up and allow reuse", () => { cache.queueWrite("root-1", "span-1", { span_id: "span-1" }); expect(cache.size).toBe(1); diff --git a/js/src/span-cache.ts b/js/src/span-cache.ts index c8a0e7d8a..c66a3d1d4 100644 --- a/js/src/span-cache.ts +++ b/js/src/span-cache.ts @@ -72,6 +72,7 @@ export class SpanCache { private fileHandle: any | null = null; // type-erased fs.promises.FileHandle private initialized = false; private initPromise: Promise | null = null; + private generation = 0; // Tracks whether the cache was explicitly disabled (via constructor or disable()) private _explicitlyDisabled: boolean; // Tracks whether the cache has been enabled (for evals only) @@ -142,6 +143,7 @@ export class SpanCache { return this.initPromise; } + const generation = this.generation; this.initPromise = (async () => { if (!iso.tmpdir || !iso.pathJoin || !iso.openFile) { // Filesystem not available - silently skip initialization @@ -150,13 +152,31 @@ export class SpanCache { const tmpDir = iso.tmpdir(); const uniqueId = `${Date.now()}-${Math.random().toString(36).slice(2)}`; - this.cacheFilePath = iso.pathJoin( + const cacheFilePath = iso.pathJoin( tmpDir, `braintrust-span-cache-${uniqueId}.jsonl`, ); // Open file for append+read - this.fileHandle = await iso.openFile(this.cacheFilePath, "a+"); + const fileHandle = await iso.openFile(cacheFilePath, "a+"); + if (generation !== this.generation) { + try { + await fileHandle.close(); + } catch { + // Ignore cleanup errors after disposal. + } + if (iso.unlinkSync) { + try { + iso.unlinkSync(cacheFilePath); + } catch { + // Ignore cleanup errors after disposal. + } + } + return; + } + + this.cacheFilePath = cacheFilePath; + this.fileHandle = fileHandle; this.initialized = true; // Register cleanup handler on first initialization @@ -245,6 +265,7 @@ export class SpanCache { * Called automatically after queueWrite, but can also be called explicitly. */ async flushWriteBuffer(): Promise { + const generation = this.generation; // Take a snapshot of records to flush, but DON'T clear the buffer yet. // Records stay in writeBuffer until disk write succeeds so getByRootSpanId can find them. const recordsToFlush = [...this.writeBuffer]; @@ -256,7 +277,7 @@ export class SpanCache { await this.ensureInitialized(); - if (!this.fileHandle) { + if (generation !== this.generation || !this.fileHandle) { return; } @@ -402,6 +423,7 @@ export class SpanCache { // Remove from global registry activeCaches.delete(this); + this.generation++; // Clear pending writes this.writeBuffer = []; From 7207156ac7c40f0866fd0054f282674caff2569e Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Thu, 17 Sep 2026 12:18:01 +0000 Subject: [PATCH 3/4] fix: conform TypeSafe traces to question spans --- .../typesafe-v0-latest-wrapped.span-tree.json | 104 ++++++++++-------- .../typesafe-v0-latest-wrapped.span-tree.txt | 104 ++++++++++-------- .../typesafe-v0-latest.span-tree.json | 104 ++++++++++-------- .../typesafe-v0-latest.span-tree.txt | 104 ++++++++++-------- .../typesafe-v0-wrapped.span-tree.json | 104 ++++++++++-------- .../typesafe-v0-wrapped.span-tree.txt | 104 ++++++++++-------- .../__snapshots__/typesafe-v0.span-tree.json | 104 ++++++++++-------- .../__snapshots__/typesafe-v0.span-tree.txt | 104 ++++++++++-------- .../typesafe-instrumentation/assertions.ts | 52 +++++---- .../plugins/typesafe-plugin.ts | 21 +++- js/src/wrappers/typesafe.test.ts | 17 ++- 11 files changed, 522 insertions(+), 400 deletions(-) diff --git a/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest-wrapped.span-tree.json b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest-wrapped.span-tree.json index 8c3e187d2..c9422cdf5 100644 --- a/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest-wrapped.span-tree.json +++ b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest-wrapped.span-tree.json @@ -9,69 +9,77 @@ "children": [ { "name": "typesafe.systemOne", - "type": "llm", + "type": "question", "children": [], "input": { - "questions": { - "category": { + "questions": [ + { "criteria": { "billing": null, "other": null, "technical": null }, + "id": "", "instructions": "Which team should handle this?", "type": "choice" }, - "duplicate_charge": { - "instructions": { - "question": "Does the customer report a duplicate charge?" - }, - "type": "noul" - }, - "urgency": { + { "criteria": [ "routine", "soon", "urgent" ], + "id": "", "instructions": "How urgent is this request?", "type": "score" + }, + { + "id": "", + "instructions": { + "question": "Does the customer report a duplicate charge?" + }, + "type": "noul" } - }, + ], "state": { "message": "I was charged twice. Please refund the duplicate charge today." } }, "output": { - "category": { - "choice": "billing", - "confidence": 1, - "probabilities": { - "billing": 1, - "other": 0, - "technical": 0 - }, - "type": "choice" - }, - "duplicate_charge": { - "noul": 0.99, - "type": "noul" - }, - "urgency": { - "confidence": 0.66, - "legend": { - "0": "routine", - "1": "soon", - "2": "urgent" + "answers": [ + { + "choice": "billing", + "confidence": 1, + "id": "", + "probabilities": { + "billing": 1, + "other": 0, + "technical": 0 + }, + "type": "choice" }, - "probabilities": { - "0": 0, - "1": 0.22, - "2": 0.78 + { + "id": "", + "noul": 0.99, + "type": "noul" }, - "score": 1.78, - "type": "score" - } + { + "confidence": 0.66, + "id": "", + "legend": { + "0": "routine", + "1": "soon", + "2": "urgent" + }, + "probabilities": { + "0": 0, + "1": 0.22, + "2": 0.78 + }, + "score": 1.78, + "type": "score" + } + ] }, "metadata": { "model": "jev-1.13.0", @@ -94,22 +102,26 @@ "children": [ { "name": "typesafe.systemOne", - "type": "llm", + "type": "question", "children": [], "input": { - "questions": { - "positive": { + "questions": [ + { + "id": "", "instructions": "Is this feedback positive?", "type": "noul" } - }, + ], "state": "The package arrived intact and on time." }, "output": { - "positive": { - "noul": 0.97, - "type": "noul" - } + "answers": [ + { + "id": "", + "noul": 0.97, + "type": "noul" + } + ] }, "metadata": { "model": "jev-1.13.0", diff --git a/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest-wrapped.span-tree.txt b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest-wrapped.span-tree.txt index 1e4a6baac..ca95b0309 100644 --- a/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest-wrapped.span-tree.txt @@ -9,68 +9,76 @@ span_tree: │ "operation": "mixed", │ "testRunId": "" │ } - │ └── typesafe.systemOne [llm] + │ └── typesafe.systemOne [question] │ input: { - │ "questions": { - │ "category": { + │ "questions": [ + │ { │ "criteria": { │ "billing": null, │ "other": null, │ "technical": null │ }, + │ "id": "", │ "instructions": "Which team should handle this?", │ "type": "choice" │ }, - │ "duplicate_charge": { - │ "instructions": { - │ "question": "Does the customer report a duplicate charge?" - │ }, - │ "type": "noul" - │ }, - │ "urgency": { + │ { │ "criteria": [ │ "routine", │ "soon", │ "urgent" │ ], + │ "id": "", │ "instructions": "How urgent is this request?", │ "type": "score" + │ }, + │ { + │ "id": "", + │ "instructions": { + │ "question": "Does the customer report a duplicate charge?" + │ }, + │ "type": "noul" │ } - │ }, + │ ], │ "state": { │ "message": "I was charged twice. Please refund the duplicate charge today." │ } │ } │ output: { - │ "category": { - │ "choice": "billing", - │ "confidence": 1, - │ "probabilities": { - │ "billing": 1, - │ "other": 0, - │ "technical": 0 - │ }, - │ "type": "choice" - │ }, - │ "duplicate_charge": { - │ "noul": 0.99, - │ "type": "noul" - │ }, - │ "urgency": { - │ "confidence": 0.66, - │ "legend": { - │ "0": "routine", - │ "1": "soon", - │ "2": "urgent" + │ "answers": [ + │ { + │ "choice": "billing", + │ "confidence": 1, + │ "id": "", + │ "probabilities": { + │ "billing": 1, + │ "other": 0, + │ "technical": 0 + │ }, + │ "type": "choice" │ }, - │ "probabilities": { - │ "0": 0, - │ "1": 0.22, - │ "2": 0.78 + │ { + │ "id": "", + │ "noul": 0.99, + │ "type": "noul" │ }, - │ "score": 1.78, - │ "type": "score" - │ } + │ { + │ "confidence": 0.66, + │ "id": "", + │ "legend": { + │ "0": "routine", + │ "1": "soon", + │ "2": "urgent" + │ }, + │ "probabilities": { + │ "0": 0, + │ "1": 0.22, + │ "2": 0.78 + │ }, + │ "score": 1.78, + │ "type": "score" + │ } + │ ] │ } │ metadata: { │ "model": "jev-1.13.0", @@ -86,21 +94,25 @@ span_tree: "operation": "raw", "testRunId": "" } - └── typesafe.systemOne [llm] + └── typesafe.systemOne [question] input: { - "questions": { - "positive": { + "questions": [ + { + "id": "", "instructions": "Is this feedback positive?", "type": "noul" } - }, + ], "state": "The package arrived intact and on time." } output: { - "positive": { - "noul": 0.97, - "type": "noul" - } + "answers": [ + { + "id": "", + "noul": 0.97, + "type": "noul" + } + ] } metadata: { "model": "jev-1.13.0", diff --git a/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest.span-tree.json b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest.span-tree.json index 8c3e187d2..c9422cdf5 100644 --- a/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest.span-tree.json +++ b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest.span-tree.json @@ -9,69 +9,77 @@ "children": [ { "name": "typesafe.systemOne", - "type": "llm", + "type": "question", "children": [], "input": { - "questions": { - "category": { + "questions": [ + { "criteria": { "billing": null, "other": null, "technical": null }, + "id": "", "instructions": "Which team should handle this?", "type": "choice" }, - "duplicate_charge": { - "instructions": { - "question": "Does the customer report a duplicate charge?" - }, - "type": "noul" - }, - "urgency": { + { "criteria": [ "routine", "soon", "urgent" ], + "id": "", "instructions": "How urgent is this request?", "type": "score" + }, + { + "id": "", + "instructions": { + "question": "Does the customer report a duplicate charge?" + }, + "type": "noul" } - }, + ], "state": { "message": "I was charged twice. Please refund the duplicate charge today." } }, "output": { - "category": { - "choice": "billing", - "confidence": 1, - "probabilities": { - "billing": 1, - "other": 0, - "technical": 0 - }, - "type": "choice" - }, - "duplicate_charge": { - "noul": 0.99, - "type": "noul" - }, - "urgency": { - "confidence": 0.66, - "legend": { - "0": "routine", - "1": "soon", - "2": "urgent" + "answers": [ + { + "choice": "billing", + "confidence": 1, + "id": "", + "probabilities": { + "billing": 1, + "other": 0, + "technical": 0 + }, + "type": "choice" }, - "probabilities": { - "0": 0, - "1": 0.22, - "2": 0.78 + { + "id": "", + "noul": 0.99, + "type": "noul" }, - "score": 1.78, - "type": "score" - } + { + "confidence": 0.66, + "id": "", + "legend": { + "0": "routine", + "1": "soon", + "2": "urgent" + }, + "probabilities": { + "0": 0, + "1": 0.22, + "2": 0.78 + }, + "score": 1.78, + "type": "score" + } + ] }, "metadata": { "model": "jev-1.13.0", @@ -94,22 +102,26 @@ "children": [ { "name": "typesafe.systemOne", - "type": "llm", + "type": "question", "children": [], "input": { - "questions": { - "positive": { + "questions": [ + { + "id": "", "instructions": "Is this feedback positive?", "type": "noul" } - }, + ], "state": "The package arrived intact and on time." }, "output": { - "positive": { - "noul": 0.97, - "type": "noul" - } + "answers": [ + { + "id": "", + "noul": 0.97, + "type": "noul" + } + ] }, "metadata": { "model": "jev-1.13.0", diff --git a/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest.span-tree.txt b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest.span-tree.txt index 1e4a6baac..ca95b0309 100644 --- a/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest.span-tree.txt +++ b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-latest.span-tree.txt @@ -9,68 +9,76 @@ span_tree: │ "operation": "mixed", │ "testRunId": "" │ } - │ └── typesafe.systemOne [llm] + │ └── typesafe.systemOne [question] │ input: { - │ "questions": { - │ "category": { + │ "questions": [ + │ { │ "criteria": { │ "billing": null, │ "other": null, │ "technical": null │ }, + │ "id": "", │ "instructions": "Which team should handle this?", │ "type": "choice" │ }, - │ "duplicate_charge": { - │ "instructions": { - │ "question": "Does the customer report a duplicate charge?" - │ }, - │ "type": "noul" - │ }, - │ "urgency": { + │ { │ "criteria": [ │ "routine", │ "soon", │ "urgent" │ ], + │ "id": "", │ "instructions": "How urgent is this request?", │ "type": "score" + │ }, + │ { + │ "id": "", + │ "instructions": { + │ "question": "Does the customer report a duplicate charge?" + │ }, + │ "type": "noul" │ } - │ }, + │ ], │ "state": { │ "message": "I was charged twice. Please refund the duplicate charge today." │ } │ } │ output: { - │ "category": { - │ "choice": "billing", - │ "confidence": 1, - │ "probabilities": { - │ "billing": 1, - │ "other": 0, - │ "technical": 0 - │ }, - │ "type": "choice" - │ }, - │ "duplicate_charge": { - │ "noul": 0.99, - │ "type": "noul" - │ }, - │ "urgency": { - │ "confidence": 0.66, - │ "legend": { - │ "0": "routine", - │ "1": "soon", - │ "2": "urgent" + │ "answers": [ + │ { + │ "choice": "billing", + │ "confidence": 1, + │ "id": "", + │ "probabilities": { + │ "billing": 1, + │ "other": 0, + │ "technical": 0 + │ }, + │ "type": "choice" │ }, - │ "probabilities": { - │ "0": 0, - │ "1": 0.22, - │ "2": 0.78 + │ { + │ "id": "", + │ "noul": 0.99, + │ "type": "noul" │ }, - │ "score": 1.78, - │ "type": "score" - │ } + │ { + │ "confidence": 0.66, + │ "id": "", + │ "legend": { + │ "0": "routine", + │ "1": "soon", + │ "2": "urgent" + │ }, + │ "probabilities": { + │ "0": 0, + │ "1": 0.22, + │ "2": 0.78 + │ }, + │ "score": 1.78, + │ "type": "score" + │ } + │ ] │ } │ metadata: { │ "model": "jev-1.13.0", @@ -86,21 +94,25 @@ span_tree: "operation": "raw", "testRunId": "" } - └── typesafe.systemOne [llm] + └── typesafe.systemOne [question] input: { - "questions": { - "positive": { + "questions": [ + { + "id": "", "instructions": "Is this feedback positive?", "type": "noul" } - }, + ], "state": "The package arrived intact and on time." } output: { - "positive": { - "noul": 0.97, - "type": "noul" - } + "answers": [ + { + "id": "", + "noul": 0.97, + "type": "noul" + } + ] } metadata: { "model": "jev-1.13.0", diff --git a/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-wrapped.span-tree.json b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-wrapped.span-tree.json index fe750162e..9b1e5681c 100644 --- a/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-wrapped.span-tree.json +++ b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-wrapped.span-tree.json @@ -9,69 +9,77 @@ "children": [ { "name": "typesafe.systemOne", - "type": "llm", + "type": "question", "children": [], "input": { - "questions": { - "category": { + "questions": [ + { "criteria": { "billing": null, "other": null, "technical": null }, + "id": "", "instructions": "Which team should handle this?", "type": "choice" }, - "duplicate_charge": { - "instructions": { - "question": "Does the customer report a duplicate charge?" - }, - "type": "noul" - }, - "urgency": { + { "criteria": [ "routine", "soon", "urgent" ], + "id": "", "instructions": "How urgent is this request?", "type": "score" + }, + { + "id": "", + "instructions": { + "question": "Does the customer report a duplicate charge?" + }, + "type": "noul" } - }, + ], "state": { "message": "I was charged twice. Please refund the duplicate charge today." } }, "output": { - "category": { - "choice": "billing", - "confidence": 1, - "probabilities": { - "billing": 1, - "other": 0, - "technical": 0 - }, - "type": "choice" - }, - "duplicate_charge": { - "noul": 0.99, - "type": "noul" - }, - "urgency": { - "confidence": 0.71, - "legend": { - "0": "routine", - "1": "soon", - "2": "urgent" + "answers": [ + { + "choice": "billing", + "confidence": 1, + "id": "", + "probabilities": { + "billing": 1, + "other": 0, + "technical": 0 + }, + "type": "choice" }, - "probabilities": { - "0": 0, - "1": 0.19, - "2": 0.81 + { + "id": "", + "noul": 0.99, + "type": "noul" }, - "score": 1.8, - "type": "score" - } + { + "confidence": 0.71, + "id": "", + "legend": { + "0": "routine", + "1": "soon", + "2": "urgent" + }, + "probabilities": { + "0": 0, + "1": 0.19, + "2": 0.81 + }, + "score": 1.8, + "type": "score" + } + ] }, "metadata": { "model": "jev-1.13.0", @@ -94,22 +102,26 @@ "children": [ { "name": "typesafe.systemOne", - "type": "llm", + "type": "question", "children": [], "input": { - "questions": { - "positive": { + "questions": [ + { + "id": "", "instructions": "Is this feedback positive?", "type": "noul" } - }, + ], "state": "The package arrived intact and on time." }, "output": { - "positive": { - "noul": 0.97, - "type": "noul" - } + "answers": [ + { + "id": "", + "noul": 0.97, + "type": "noul" + } + ] }, "metadata": { "model": "jev-1.13.0", diff --git a/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-wrapped.span-tree.txt b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-wrapped.span-tree.txt index 1b38a474e..d853f3de2 100644 --- a/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-wrapped.span-tree.txt +++ b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0-wrapped.span-tree.txt @@ -9,68 +9,76 @@ span_tree: │ "operation": "mixed", │ "testRunId": "" │ } - │ └── typesafe.systemOne [llm] + │ └── typesafe.systemOne [question] │ input: { - │ "questions": { - │ "category": { + │ "questions": [ + │ { │ "criteria": { │ "billing": null, │ "other": null, │ "technical": null │ }, + │ "id": "", │ "instructions": "Which team should handle this?", │ "type": "choice" │ }, - │ "duplicate_charge": { - │ "instructions": { - │ "question": "Does the customer report a duplicate charge?" - │ }, - │ "type": "noul" - │ }, - │ "urgency": { + │ { │ "criteria": [ │ "routine", │ "soon", │ "urgent" │ ], + │ "id": "", │ "instructions": "How urgent is this request?", │ "type": "score" + │ }, + │ { + │ "id": "", + │ "instructions": { + │ "question": "Does the customer report a duplicate charge?" + │ }, + │ "type": "noul" │ } - │ }, + │ ], │ "state": { │ "message": "I was charged twice. Please refund the duplicate charge today." │ } │ } │ output: { - │ "category": { - │ "choice": "billing", - │ "confidence": 1, - │ "probabilities": { - │ "billing": 1, - │ "other": 0, - │ "technical": 0 - │ }, - │ "type": "choice" - │ }, - │ "duplicate_charge": { - │ "noul": 0.99, - │ "type": "noul" - │ }, - │ "urgency": { - │ "confidence": 0.71, - │ "legend": { - │ "0": "routine", - │ "1": "soon", - │ "2": "urgent" + │ "answers": [ + │ { + │ "choice": "billing", + │ "confidence": 1, + │ "id": "", + │ "probabilities": { + │ "billing": 1, + │ "other": 0, + │ "technical": 0 + │ }, + │ "type": "choice" │ }, - │ "probabilities": { - │ "0": 0, - │ "1": 0.19, - │ "2": 0.81 + │ { + │ "id": "", + │ "noul": 0.99, + │ "type": "noul" │ }, - │ "score": 1.8, - │ "type": "score" - │ } + │ { + │ "confidence": 0.71, + │ "id": "", + │ "legend": { + │ "0": "routine", + │ "1": "soon", + │ "2": "urgent" + │ }, + │ "probabilities": { + │ "0": 0, + │ "1": 0.19, + │ "2": 0.81 + │ }, + │ "score": 1.8, + │ "type": "score" + │ } + │ ] │ } │ metadata: { │ "model": "jev-1.13.0", @@ -86,21 +94,25 @@ span_tree: "operation": "raw", "testRunId": "" } - └── typesafe.systemOne [llm] + └── typesafe.systemOne [question] input: { - "questions": { - "positive": { + "questions": [ + { + "id": "", "instructions": "Is this feedback positive?", "type": "noul" } - }, + ], "state": "The package arrived intact and on time." } output: { - "positive": { - "noul": 0.97, - "type": "noul" - } + "answers": [ + { + "id": "", + "noul": 0.97, + "type": "noul" + } + ] } metadata: { "model": "jev-1.13.0", diff --git a/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0.span-tree.json b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0.span-tree.json index fe750162e..9b1e5681c 100644 --- a/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0.span-tree.json +++ b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0.span-tree.json @@ -9,69 +9,77 @@ "children": [ { "name": "typesafe.systemOne", - "type": "llm", + "type": "question", "children": [], "input": { - "questions": { - "category": { + "questions": [ + { "criteria": { "billing": null, "other": null, "technical": null }, + "id": "", "instructions": "Which team should handle this?", "type": "choice" }, - "duplicate_charge": { - "instructions": { - "question": "Does the customer report a duplicate charge?" - }, - "type": "noul" - }, - "urgency": { + { "criteria": [ "routine", "soon", "urgent" ], + "id": "", "instructions": "How urgent is this request?", "type": "score" + }, + { + "id": "", + "instructions": { + "question": "Does the customer report a duplicate charge?" + }, + "type": "noul" } - }, + ], "state": { "message": "I was charged twice. Please refund the duplicate charge today." } }, "output": { - "category": { - "choice": "billing", - "confidence": 1, - "probabilities": { - "billing": 1, - "other": 0, - "technical": 0 - }, - "type": "choice" - }, - "duplicate_charge": { - "noul": 0.99, - "type": "noul" - }, - "urgency": { - "confidence": 0.71, - "legend": { - "0": "routine", - "1": "soon", - "2": "urgent" + "answers": [ + { + "choice": "billing", + "confidence": 1, + "id": "", + "probabilities": { + "billing": 1, + "other": 0, + "technical": 0 + }, + "type": "choice" }, - "probabilities": { - "0": 0, - "1": 0.19, - "2": 0.81 + { + "id": "", + "noul": 0.99, + "type": "noul" }, - "score": 1.8, - "type": "score" - } + { + "confidence": 0.71, + "id": "", + "legend": { + "0": "routine", + "1": "soon", + "2": "urgent" + }, + "probabilities": { + "0": 0, + "1": 0.19, + "2": 0.81 + }, + "score": 1.8, + "type": "score" + } + ] }, "metadata": { "model": "jev-1.13.0", @@ -94,22 +102,26 @@ "children": [ { "name": "typesafe.systemOne", - "type": "llm", + "type": "question", "children": [], "input": { - "questions": { - "positive": { + "questions": [ + { + "id": "", "instructions": "Is this feedback positive?", "type": "noul" } - }, + ], "state": "The package arrived intact and on time." }, "output": { - "positive": { - "noul": 0.97, - "type": "noul" - } + "answers": [ + { + "id": "", + "noul": 0.97, + "type": "noul" + } + ] }, "metadata": { "model": "jev-1.13.0", diff --git a/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0.span-tree.txt b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0.span-tree.txt index 1b38a474e..d853f3de2 100644 --- a/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0.span-tree.txt +++ b/e2e/scenarios/typesafe-instrumentation/__snapshots__/typesafe-v0.span-tree.txt @@ -9,68 +9,76 @@ span_tree: │ "operation": "mixed", │ "testRunId": "" │ } - │ └── typesafe.systemOne [llm] + │ └── typesafe.systemOne [question] │ input: { - │ "questions": { - │ "category": { + │ "questions": [ + │ { │ "criteria": { │ "billing": null, │ "other": null, │ "technical": null │ }, + │ "id": "", │ "instructions": "Which team should handle this?", │ "type": "choice" │ }, - │ "duplicate_charge": { - │ "instructions": { - │ "question": "Does the customer report a duplicate charge?" - │ }, - │ "type": "noul" - │ }, - │ "urgency": { + │ { │ "criteria": [ │ "routine", │ "soon", │ "urgent" │ ], + │ "id": "", │ "instructions": "How urgent is this request?", │ "type": "score" + │ }, + │ { + │ "id": "", + │ "instructions": { + │ "question": "Does the customer report a duplicate charge?" + │ }, + │ "type": "noul" │ } - │ }, + │ ], │ "state": { │ "message": "I was charged twice. Please refund the duplicate charge today." │ } │ } │ output: { - │ "category": { - │ "choice": "billing", - │ "confidence": 1, - │ "probabilities": { - │ "billing": 1, - │ "other": 0, - │ "technical": 0 - │ }, - │ "type": "choice" - │ }, - │ "duplicate_charge": { - │ "noul": 0.99, - │ "type": "noul" - │ }, - │ "urgency": { - │ "confidence": 0.71, - │ "legend": { - │ "0": "routine", - │ "1": "soon", - │ "2": "urgent" + │ "answers": [ + │ { + │ "choice": "billing", + │ "confidence": 1, + │ "id": "", + │ "probabilities": { + │ "billing": 1, + │ "other": 0, + │ "technical": 0 + │ }, + │ "type": "choice" │ }, - │ "probabilities": { - │ "0": 0, - │ "1": 0.19, - │ "2": 0.81 + │ { + │ "id": "", + │ "noul": 0.99, + │ "type": "noul" │ }, - │ "score": 1.8, - │ "type": "score" - │ } + │ { + │ "confidence": 0.71, + │ "id": "", + │ "legend": { + │ "0": "routine", + │ "1": "soon", + │ "2": "urgent" + │ }, + │ "probabilities": { + │ "0": 0, + │ "1": 0.19, + │ "2": 0.81 + │ }, + │ "score": 1.8, + │ "type": "score" + │ } + │ ] │ } │ metadata: { │ "model": "jev-1.13.0", @@ -86,21 +94,25 @@ span_tree: "operation": "raw", "testRunId": "" } - └── typesafe.systemOne [llm] + └── typesafe.systemOne [question] input: { - "questions": { - "positive": { + "questions": [ + { + "id": "", "instructions": "Is this feedback positive?", "type": "noul" } - }, + ], "state": "The package arrived intact and on time." } output: { - "positive": { - "noul": 0.97, - "type": "noul" - } + "answers": [ + { + "id": "", + "noul": 0.97, + "type": "noul" + } + ] } metadata: { "model": "jev-1.13.0", diff --git a/e2e/scenarios/typesafe-instrumentation/assertions.ts b/e2e/scenarios/typesafe-instrumentation/assertions.ts index 9653a6576..f49d686ce 100644 --- a/e2e/scenarios/typesafe-instrumentation/assertions.ts +++ b/e2e/scenarios/typesafe-instrumentation/assertions.ts @@ -74,30 +74,38 @@ export function defineTypeSafeInstrumentationAssertions(options: { ); expect(root?.row.metadata).toMatchObject({ scenario: SCENARIO_NAME }); - expect(span?.span.type).toBe("llm"); + expect(span?.span.type).toBe("question"); expect(span?.input).toMatchObject({ state: { message: expect.any(String) }, - questions: { - category: { type: "choice" }, - urgency: { type: "score" }, - duplicate_charge: { type: "noul" }, - }, + questions: expect.arrayContaining([ + expect.objectContaining({ id: "category", type: "choice" }), + expect.objectContaining({ id: "urgency", type: "score" }), + expect.objectContaining({ id: "duplicate_charge", type: "noul" }), + ]), }); expect(span?.output).toMatchObject({ - category: { - type: "choice", - choice: expect.any(String), - confidence: expect.any(Number), - probabilities: expect.any(Object), - }, - urgency: { - type: "score", - score: expect.any(Number), - confidence: expect.any(Number), - legend: expect.any(Object), - probabilities: expect.any(Object), - }, - duplicate_charge: { type: "noul", noul: expect.any(Number) }, + answers: expect.arrayContaining([ + expect.objectContaining({ + id: "category", + type: "choice", + choice: expect.any(String), + confidence: expect.any(Number), + probabilities: expect.any(Object), + }), + expect.objectContaining({ + id: "urgency", + type: "score", + score: expect.any(Number), + confidence: expect.any(Number), + legend: expect.any(Object), + probabilities: expect.any(Object), + }), + expect.objectContaining({ + id: "duplicate_charge", + type: "noul", + noul: expect.any(Number), + }), + ]), }); expect(span?.row.metadata).toMatchObject({ model: expect.stringMatching(/^jev-/), @@ -120,10 +128,10 @@ export function defineTypeSafeInstrumentationAssertions(options: { expect(span?.input).toMatchObject({ state: "The package arrived intact and on time.", - questions: { positive: { type: "noul" } }, + questions: [{ id: "positive", type: "noul" }], }); expect(span?.output).toMatchObject({ - positive: { type: "noul", noul: expect.any(Number) }, + answers: [{ id: "positive", type: "noul", noul: expect.any(Number) }], }); expect(span?.row.metadata).toMatchObject({ model: "jev-1.13.0", diff --git a/js/src/instrumentation/plugins/typesafe-plugin.ts b/js/src/instrumentation/plugins/typesafe-plugin.ts index 7bc8d6a47..af0547f2c 100644 --- a/js/src/instrumentation/plugins/typesafe-plugin.ts +++ b/js/src/instrumentation/plugins/typesafe-plugin.ts @@ -1,4 +1,4 @@ -import { SpanTypeAttribute, isObject } from "../../../util/index"; +import { isObject } from "../../../util/index"; import { debugLogger } from "../../debug-logger"; import { startSpan, withCurrent } from "../../logger"; import type { Span } from "../../logger"; @@ -62,7 +62,7 @@ function interceptSystemOne( event: { input: { state: request?.state, - questions: request?.questions, + questions: addIds(request?.questions), }, metadata: { ...(model ? { model } : {}), @@ -70,7 +70,7 @@ function interceptSystemOne( }, }, name: "typesafe.systemOne", - spanAttributes: { type: SpanTypeAttribute.LLM }, + spanAttributes: { type: "question" }, }, INSTRUMENTATION_NAMES.TYPESAFE, ), @@ -225,7 +225,10 @@ function finishSuccessfulSpan( finishTypeSafeSpan(span, () => { const metrics = extractMetrics(result); span.log({ - output: isObject(result) ? result.answers : undefined, + output: + isObject(result) && result.answers !== undefined + ? { answers: addIds(result.answers) } + : undefined, ...(isObject(result) && typeof result.model === "string" ? { metadata: { model: result.model } } : {}), @@ -234,6 +237,16 @@ function finishSuccessfulSpan( }); } +function addIds(value: unknown): unknown { + if (!isObject(value)) { + return value; + } + + return Object.entries(value).map(([id, entry]) => + isObject(entry) ? { ...entry, id } : { id, value: entry }, + ); +} + function extractMetrics( result: TypeSafeSystemOneResult, ): Record { diff --git a/js/src/wrappers/typesafe.test.ts b/js/src/wrappers/typesafe.test.ts index 4ed46740c..33e9d548d 100644 --- a/js/src/wrappers/typesafe.test.ts +++ b/js/src/wrappers/typesafe.test.ts @@ -154,16 +154,21 @@ describe("TypeSafe wrapper", () => { ) as Record | undefined; expect(span).toMatchObject({ input: { - questions: { - category: { type: "choice" }, - urgent: { type: "noul" }, - }, + questions: [ + { id: "category", type: "choice" }, + { id: "urgent", type: "noul" }, + ], state: { message: "Charged twice" }, }, metadata: { model: "jev-1.13.0", provider: "typesafe" }, metrics: { completion_tokens: 5, prompt_tokens: 20, tokens: 25 }, - output: result.answers, - span_attributes: { name: "typesafe.systemOne", type: "llm" }, + output: { + answers: [ + { ...result.answers.category, id: "category" }, + { ...result.answers.urgent, id: "urgent" }, + ], + }, + span_attributes: { name: "typesafe.systemOne", type: "question" }, }); }); From cc60c3010e5f10618266f7dcc715c1b3b054c2eb Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Thu, 17 Sep 2026 13:27:03 +0000 Subject: [PATCH 4/4] Update PR #2493 --- .agents/skills/instrumentation/SKILL.md | 3 +- e2e/helpers/scenario-installer.ts | 1 + e2e/scripts/bump-e2e-versions.mjs | 1 + .../plugins/typesafe-plugin.ts | 51 ++++++++----------- 4 files changed, 26 insertions(+), 30 deletions(-) diff --git a/.agents/skills/instrumentation/SKILL.md b/.agents/skills/instrumentation/SKILL.md index 35bf300aa..1404b5c06 100644 --- a/.agents/skills/instrumentation/SKILL.md +++ b/.agents/skills/instrumentation/SKILL.md @@ -34,7 +34,8 @@ Map the change before editing: - Maintain isomorphic behavior. Node and browser/bundled paths must use compatible channel implementations and avoid channel-registry mismatches. - Setup, teardown, and patching must be idempotent. Enabling twice, disabling twice, or applying a patch twice should remain safe. - Promise/stream behavior must be preserved. Patches need to keep subclass/helper semantics intact. -- Contain instrumentation failures. Extraction/logging bugs should be logged or ignored as appropriate, but must not break the user call path. +- Treat Braintrust logging and span-finalization methods, including `span.log()` and `span.end()`, as non-throwing. + Do not wrap these methods in defensive `try`/`catch` blocks or add fallback logic for hypothetical logging exceptions. - Use the SDK `debugLogger` for SDK instrumentation diagnostics. Do not call `console.*` directly from instrumentation code; direct console use should stay inside the debug logger implementation or another explicitly justified exception. - Pass `Error` objects directly as `span.log({ error })` values. The SDK serializes errors correctly, so do not add local helpers that manually turn errors into message/stack strings unless an external API requires a non-`Error` representation. - Log only the useful surface. Prefer narrow, stable payloads over dumping full request/response objects; exclude redundant or overly large data when possible. diff --git a/e2e/helpers/scenario-installer.ts b/e2e/helpers/scenario-installer.ts index bcb254546..5f43d09a4 100644 --- a/e2e/helpers/scenario-installer.ts +++ b/e2e/helpers/scenario-installer.ts @@ -32,6 +32,7 @@ const INSTALL_SECRET_ENV_VARS = [ "OPENAI_API_KEY", "OPENROUTER_API_KEY", "MISTRAL_API_KEY", + "TYPESAFE_API_KEY", ] as const; const cleanupDirs = new Set(); diff --git a/e2e/scripts/bump-e2e-versions.mjs b/e2e/scripts/bump-e2e-versions.mjs index 35d006ed1..d7ed675a1 100644 --- a/e2e/scripts/bump-e2e-versions.mjs +++ b/e2e/scripts/bump-e2e-versions.mjs @@ -32,6 +32,7 @@ const INSTALL_SECRET_ENV_VARS = [ "MISTRAL_API_KEY", "OPENAI_API_KEY", "OPENROUTER_API_KEY", + "TYPESAFE_API_KEY", ]; const rawArgs = process.argv.slice(2).filter((arg) => arg !== "--"); diff --git a/js/src/instrumentation/plugins/typesafe-plugin.ts b/js/src/instrumentation/plugins/typesafe-plugin.ts index af0547f2c..aee7b5911 100644 --- a/js/src/instrumentation/plugins/typesafe-plugin.ts +++ b/js/src/instrumentation/plugins/typesafe-plugin.ts @@ -86,14 +86,18 @@ function interceptSystemOne( runWithAutoInstrumentationSuppressed(invokeTarget), ); } catch (error) { - finishTypeSafeSpan(span, () => span.log({ error })); + span.log({ error }); + span.end(); throw error; } if (!isTypeSafeAPIPromise(result)) { const finished = Promise.resolve(result).then( (value) => finishSuccessfulSpan(span, value), - (error) => finishTypeSafeSpan(span, () => span.log({ error })), + (error) => { + span.log({ error }); + span.end(); + }, ); void finished; return result; @@ -113,10 +117,13 @@ function interceptSystemOne( "Error reading response for typesafe.systemOne:", error, ); - finishTypeSafeSpan(span, () => {}); + span.end(); } }, - (error) => finishTypeSafeSpan(span, () => span.log({ error })), + (error) => { + span.log({ error }); + span.end(); + }, ); preserveTypeSafePromise(result, captureFinished); return result; @@ -222,19 +229,18 @@ function finishSuccessfulSpan( span: Span, result: TypeSafeSystemOneResult, ): void { - finishTypeSafeSpan(span, () => { - const metrics = extractMetrics(result); - span.log({ - output: - isObject(result) && result.answers !== undefined - ? { answers: addIds(result.answers) } - : undefined, - ...(isObject(result) && typeof result.model === "string" - ? { metadata: { model: result.model } } - : {}), - ...(Object.keys(metrics).length > 0 ? { metrics } : {}), - }); + const metrics = extractMetrics(result); + span.log({ + output: + isObject(result) && result.answers !== undefined + ? { answers: addIds(result.answers) } + : undefined, + ...(isObject(result) && typeof result.model === "string" + ? { metadata: { model: result.model } } + : {}), + ...(Object.keys(metrics).length > 0 ? { metrics } : {}), }); + span.end(); } function addIds(value: unknown): unknown { @@ -269,16 +275,3 @@ function validTokenCount(value: unknown): number | undefined { ? value : undefined; } - -function finishTypeSafeSpan(span: Span, log: () => void): void { - try { - log(); - } catch (error) { - debugLogger.error("Error logging span for typesafe.systemOne:", error); - } - try { - span.end(); - } catch (error) { - debugLogger.error("Error ending span for typesafe.systemOne:", error); - } -}