diff --git a/src/handlers/eval/ondemand/evaluate/index.tsx b/src/handlers/eval/ondemand/evaluate/index.tsx index 4d50ca7b2..6eb6f833d 100644 --- a/src/handlers/eval/ondemand/evaluate/index.tsx +++ b/src/handlers/eval/ondemand/evaluate/index.tsx @@ -8,43 +8,81 @@ import type { EvaluationReferenceInput } from "@aws-sdk/client-bedrock-agentcore import { coreOptsFromCtx, parseJsonArrayFlag } from "../../../utils"; import type { SessionWindow } from "../../types"; +const SESSION_SOURCE = "Session source:"; +const SESSION_FILTERS = "Session filters:"; +const EVALUATION = "Evaluation:"; + +const groundTruthHelp = `(JSON: list of objects) +Expected answers for the sessions being evaluated, so an evaluator can score a +response against a reference instead of judging it on its own. Each entry is an +EvaluationReferenceInput bound to a session (and optionally a trace) by its span +context; omit an entry for a session that has no reference answer. + +Accepts inline JSON, file://, or - to read stdin. + +JSON syntax: + [ + { + "context": { // [required] what this reference applies to + "spanContext": { + "sessionId": "string", // [required] + "traceId": "string" // set for a trace-level reference + } + }, + "expectedResponse": { "text": "string" }, // trace-level; needs a traceId + "assertions": [ // session-level + { "text": "string" }, + ... + ], + "expectedTrajectory": { // session-level + "toolNames": ["string", ...] // tools the agent should have called + } + }, + ... + ] + +Example: + --ground-truth '[{"context":{"spanContext":{"sessionId":"session-123"}},"assertions":[{"text":"acknowledges the shipping delay"}]}]' + + --ground-truth file://ground-truth.json`; + export const createEvaluateOnDemandHandler = (core: Core, io: AppIO) => createHandler({ name: "evaluate", description: "evaluate existing sessions client-side (synchronous; prints scores)", flags: [ - flag( - "agent", - "source: harness ID or Runtime ID whose sessions to evaluate", - z.string().optional(), - ), - flag("endpoint", "Runtime endpoint qualifier (default DEFAULT)", z.string().optional()), - flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional()), - flag( - "lookback-days", - "time filter: evaluate sessions from the last N days", - z.number().optional(), - ), - flag( - "start-time", - "time filter: window start (ISO-8601, with --end-time)", - z.string().optional(), - ), - flag( - "end-time", - "time filter: window end (ISO-8601, with --start-time)", - z.string().optional(), - ), - flag("session-ids", "filter: specific session IDs", z.array(z.string()).optional()), + flag("agent", "harness ID or Runtime ID whose sessions to evaluate", z.string().optional(), { + group: SESSION_SOURCE, + }), + flag("endpoint", "Runtime endpoint qualifier (default DEFAULT)", z.string().optional(), { + group: SESSION_SOURCE, + }), + flag("lookback-days", "evaluate sessions from the last N days", z.number().optional(), { + group: SESSION_FILTERS, + }), + flag("start-time", "window start (ISO-8601, with --end-time)", z.string().optional(), { + group: SESSION_FILTERS, + }), + flag("end-time", "window end (ISO-8601, with --start-time)", z.string().optional(), { + group: SESSION_FILTERS, + }), + flag("session-ids", "specific session IDs", z.array(z.string()).optional(), { + group: SESSION_FILTERS, + }), flag( "trace-id", - "filter: a single trace ID (session ID is read off the span)", + "a single trace ID (session ID is read off the span)", z.string().optional(), + { group: SESSION_FILTERS }, ), + flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional(), { + group: EVALUATION, + }), flag( "ground-truth", - "ground truth (JSON EvaluationReferenceInput[]; inline, file://, or -)", + "expected answers (JSON EvaluationReferenceInput[])", z.string().optional(), + { group: EVALUATION, help: groundTruthHelp }, ), ], handle: async (ctx, flags) => { diff --git a/src/handlers/eval/ondemand/simulate/index.tsx b/src/handlers/eval/ondemand/simulate/index.tsx index 17d24f3ed..22e7ec83d 100644 --- a/src/handlers/eval/ondemand/simulate/index.tsx +++ b/src/handlers/eval/ondemand/simulate/index.tsx @@ -10,35 +10,63 @@ import { coreOptsFromCtx } from "../../../utils"; import { parseRuntimeInvokeHeaders } from "../../../runtime/invoke/request"; import { withUserCancellation } from "../../../../runnable"; +const TARGET = "Target:"; +const INVOCATION = "Invocation:"; +const DATASET = "Dataset:"; +const EVALUATION = "Evaluation:"; + +const payloadTemplateHelp = `(JSON object) +The request body sent to the Runtime for each dataset example. Every occurrence +of {input} is replaced with that example's input, so the template describes the +shape your agent expects and {input} marks where the prompt goes. + +Example: + --payload-template '{"prompt":"{input}"}' + + --payload-template '{"messages":[{"role":"user","content":"{input}"}],"stream":false}'`; + export const createSimulateOnDemandHandler = (core: Core, _io: AppIO) => createHandler({ name: "simulate", description: "replay a dataset against a Runtime, then evaluate the sessions client-side", flags: [ - flag("runtime-id", "Runtime ID to invoke per scenario", z.string().optional()), - flag("endpoint", "Runtime endpoint qualifier (default DEFAULT)", z.string().optional()), + flag("runtime-id", "Runtime ID to invoke per scenario", z.string().optional(), { + group: TARGET, + }), + flag("endpoint", "Runtime endpoint qualifier (default DEFAULT)", z.string().optional(), { + group: TARGET, + }), flag( "payload-template", - 'JSON payload template; {input} is the scenario input, e.g. {"prompt":"{input}"}', + "request body per example (JSON object); {input} is replaced with the input", z.string().optional(), + { group: INVOCATION, help: payloadTemplateHelp }, ), flag("header", "an ordered application header (repeatable)", z.array(z.string()).optional(), { sensitive: true, + group: INVOCATION, }), flag( "bearer-token", "CUSTOM_JWT bearer token (for JWT-auth Runtimes)", z.string().optional(), - { sensitive: true }, + { sensitive: true, group: INVOCATION }, ), - flag("user-id", "Runtime user ID", z.string().optional()), - flag("dataset", "dataset source: local JSONL path or a dataset ID", z.string().optional()), - flag("dataset-version", "dataset version (with a dataset ID)", z.string().optional()), - flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional()), + flag("user-id", "Runtime user ID", z.string().optional(), { group: INVOCATION }), + flag("dataset", "local JSONL path or a dataset ID", z.string().optional(), { + group: DATASET, + }), + flag("dataset-version", "dataset version (with a dataset ID)", z.string().optional(), { + group: DATASET, + }), + flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional(), { + group: EVALUATION, + }), flag( "ingestion-wait-ms", "ms to wait for span ingestion before grading (default 180000; 0 to skip)", z.coerce.number().int().nonnegative().optional(), + { group: EVALUATION }, ), ], handle: async (ctx, flags) => {