feat(eval): grouped help for the ondemand commands

src/handlers/eval/ondemand/evaluate/index.tsx

@@ -8,43 +8,81 @@ import type { EvaluationReferenceInput } from "@aws-sdk/client-bedrock-agentcore
 import { coreOptsFromCtx, parseJsonArrayFlag } from "../../../utils";
 import type { SessionWindow } from "../../types";
 
+const SESSION_SOURCE = "Session source:";
+const SESSION_FILTERS = "Session filters:";
+const EVALUATION = "Evaluation:";
+
+const groundTruthHelp = `(JSON: list of objects)
+Expected answers for the sessions being evaluated, so an evaluator can score a
+response against a reference instead of judging it on its own. Each entry is an
+EvaluationReferenceInput bound to a session (and optionally a trace) by its span
+context; omit an entry for a session that has no reference answer.
+
+Accepts inline JSON, file://<path>, or - to read stdin.
+
+JSON syntax:
+  [
+    {
+      "context": {                        // [required] what this reference applies to
+        "spanContext": {
+          "sessionId": "string",          // [required]
+          "traceId": "string"             // set for a trace-level reference
+        }
+      },
+      "expectedResponse": { "text": "string" },   // trace-level; needs a traceId
+      "assertions": [                              // session-level
+        { "text": "string" },
+        ...
+      ],
+      "expectedTrajectory": {                      // session-level
+        "toolNames": ["string", ...]    // tools the agent should have called
+      }
+    },
+    ...
+  ]
+
+Example:
+  --ground-truth '[{"context":{"spanContext":{"sessionId":"session-123"}},"assertions":[{"text":"acknowledges the shipping delay"}]}]'
+
+  --ground-truth file://ground-truth.json`;
+
 export const createEvaluateOnDemandHandler = (core: Core, io: AppIO) =>
   createHandler({
     name: "evaluate",
     description: "evaluate existing sessions client-side (synchronous; prints scores)",
     flags: [
-      flag(
-        "agent",
-        "source: harness ID or Runtime ID whose sessions to evaluate",
-        z.string().optional(),
-      ),
-      flag("endpoint", "Runtime endpoint qualifier (default DEFAULT)", z.string().optional()),
-      flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional()),
-      flag(
-        "lookback-days",
-        "time filter: evaluate sessions from the last N days",
-        z.number().optional(),
-      ),
-      flag(
-        "start-time",
-        "time filter: window start (ISO-8601, with --end-time)",
-        z.string().optional(),
-      ),
-      flag(
-        "end-time",
-        "time filter: window end (ISO-8601, with --start-time)",
-        z.string().optional(),
-      ),
-      flag("session-ids", "filter: specific session IDs", z.array(z.string()).optional()),
+      flag("agent", "harness ID or Runtime ID whose sessions to evaluate", z.string().optional(), {
+        group: SESSION_SOURCE,
+      }),
+      flag("endpoint", "Runtime endpoint qualifier (default DEFAULT)", z.string().optional(), {
+        group: SESSION_SOURCE,
+      }),
+      flag("lookback-days", "evaluate sessions from the last N days", z.number().optional(), {
+        group: SESSION_FILTERS,
+      }),
+      flag("start-time", "window start (ISO-8601, with --end-time)", z.string().optional(), {
+        group: SESSION_FILTERS,
+      }),
+      flag("end-time", "window end (ISO-8601, with --start-time)", z.string().optional(), {
+        group: SESSION_FILTERS,
+      }),
+      flag("session-ids", "specific session IDs", z.array(z.string()).optional(), {
+        group: SESSION_FILTERS,
+      }),
       flag(
         "trace-id",
-        "filter: a single trace ID (session ID is read off the span)",
+        "a single trace ID (session ID is read off the span)",
         z.string().optional(),
+        { group: SESSION_FILTERS },
       ),
+      flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional(), {
+        group: EVALUATION,
+      }),
       flag(
         "ground-truth",
-        "ground truth (JSON EvaluationReferenceInput[]; inline, file://<path>, or -)",
+        "expected answers (JSON EvaluationReferenceInput[])",
         z.string().optional(),
+        { group: EVALUATION, help: groundTruthHelp },
       ),
     ],
     handle: async (ctx, flags) => {

src/handlers/eval/ondemand/simulate/index.tsx

@@ -10,35 +10,63 @@ import { coreOptsFromCtx } from "../../../utils";
 import { parseRuntimeInvokeHeaders } from "../../../runtime/invoke/request";
 import { withUserCancellation } from "../../../../runnable";
 
+const TARGET = "Target:";
+const INVOCATION = "Invocation:";
+const DATASET = "Dataset:";
+const EVALUATION = "Evaluation:";
+
+const payloadTemplateHelp = `(JSON object)
+The request body sent to the Runtime for each dataset example. Every occurrence
+of {input} is replaced with that example's input, so the template describes the
+shape your agent expects and {input} marks where the prompt goes.
+
+Example:
+  --payload-template '{"prompt":"{input}"}'
+
+  --payload-template '{"messages":[{"role":"user","content":"{input}"}],"stream":false}'`;
+
 export const createSimulateOnDemandHandler = (core: Core, _io: AppIO) =>
   createHandler({
     name: "simulate",
     description: "replay a dataset against a Runtime, then evaluate the sessions client-side",
     flags: [
-      flag("runtime-id", "Runtime ID to invoke per scenario", z.string().optional()),
-      flag("endpoint", "Runtime endpoint qualifier (default DEFAULT)", z.string().optional()),
+      flag("runtime-id", "Runtime ID to invoke per scenario", z.string().optional(), {
+        group: TARGET,
+      }),
+      flag("endpoint", "Runtime endpoint qualifier (default DEFAULT)", z.string().optional(), {
+        group: TARGET,
+      }),
       flag(
         "payload-template",
-        'JSON payload template; {input} is the scenario input, e.g. {"prompt":"{input}"}',
+        "request body per example (JSON object); {input} is replaced with the input",
         z.string().optional(),
+        { group: INVOCATION, help: payloadTemplateHelp },
       ),
       flag("header", "an ordered application header (repeatable)", z.array(z.string()).optional(), {
         sensitive: true,
+        group: INVOCATION,
       }),
       flag(
         "bearer-token",
         "CUSTOM_JWT bearer token (for JWT-auth Runtimes)",
         z.string().optional(),
-        { sensitive: true },
+        { sensitive: true, group: INVOCATION },
       ),
-      flag("user-id", "Runtime user ID", z.string().optional()),
-      flag("dataset", "dataset source: local JSONL path or a dataset ID", z.string().optional()),
-      flag("dataset-version", "dataset version (with a dataset ID)", z.string().optional()),
-      flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional()),
+      flag("user-id", "Runtime user ID", z.string().optional(), { group: INVOCATION }),
+      flag("dataset", "local JSONL path or a dataset ID", z.string().optional(), {
+        group: DATASET,
+      }),
+      flag("dataset-version", "dataset version (with a dataset ID)", z.string().optional(), {
+        group: DATASET,
+      }),
+      flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional(), {
+        group: EVALUATION,
+      }),
       flag(
         "ingestion-wait-ms",
         "ms to wait for span ingestion before grading (default 180000; 0 to skip)",
         z.coerce.number().int().nonnegative().optional(),
+        { group: EVALUATION },
       ),
     ],
     handle: async (ctx, flags) => {