diff --git a/src/components/CliOnlyScreen.test.tsx b/src/components/CliOnlyScreen.test.tsx index dce691c2b..6116dbcb9 100644 --- a/src/components/CliOnlyScreen.test.tsx +++ b/src/components/CliOnlyScreen.test.tsx @@ -170,16 +170,15 @@ describe("option help groups", () => { const headingLine = (title: string) => `\n ${title}\n`; test("a grouped command renders one section per heading, in --help order", async () => { - const r = renderScreen("/agentcore/eval/batch-evaluation/evaluate"); + // simulate is a grouped, command-line-only sibling of evaluate (which now + // has a TUI screen), so it exercises the CliOnlyScreen help-group rendering. + const r = renderScreen("/agentcore/eval/batch-evaluation/simulate"); await waitForText(r.lastFrame, "this command runs from the command line"); const frame = r.lastFrame()!; - const positions = [ - "configuration", - "session source (choose exactly one)", - "source filters", - "evaluation", - ].map((title) => frame.indexOf(headingLine(title))); + const positions = ["runtime invocation", "dataset", "configuration", "evaluation"].map( + (title) => frame.indexOf(headingLine(title)), + ); expect(positions.every((position) => position >= 0)).toBe(true); expect(positions).toEqual([...positions].sort((a, b) => a - b)); diff --git a/src/components/Root.tsx b/src/components/Root.tsx index bbbf23c5b..8cb1ac1f3 100644 --- a/src/components/Root.tsx +++ b/src/components/Root.tsx @@ -58,6 +58,7 @@ import { import { BatchEvaluationScreen } from "../handlers/eval/batch-evaluation/screen.tsx"; import { BatchEvaluationListScreen } from "../handlers/eval/batch-evaluation/list/screen.tsx"; import { BatchEvaluationGetJsonScreen } from "../handlers/eval/batch-evaluation/get/screen.tsx"; +import { BatchEvaluationEvaluateScreen } from "../handlers/eval/batch-evaluation/evaluate/screen.tsx"; import { RecommendationScreen } from "../handlers/eval/recommendation/screen.tsx"; import { RecommendationListScreen } from "../handlers/eval/recommendation/list/screen.tsx"; import { RecommendationGetJsonScreen } from "../handlers/eval/recommendation/get/screen.tsx"; @@ -612,6 +613,10 @@ export function Root({ path, ctx, core, queryClient }: RootProps) { path="agentcore/eval/batch-evaluation/list" element={} /> + } + /> {/* Bare `get` (no id) has nothing to show — send the user to the list. */} ({ + name: "", + agent: "", + lookbackDays: "7", + evaluatorIds: [], + }); + const [result, setResult] = useState(); + const set = (update: Partial) => + setValues((current) => ({ ...current, ...update })); + + // First page only for now; a paged picker is a follow-up (see design doc). + // ponytail: 100-item cap, add pagination if accounts routinely exceed it. + const evaluators = useQuery({ + queryKey: ["evaluators", opts.region], + queryFn: () => core.eval.listEvaluators(undefined, 100, opts), + }); + + return ( + navigate(MENU)} + onSubmit={async () => { + const endTime = new Date(); + const startTime = new Date(endTime.getTime() - Number(values.lookbackDays) * 86_400_000); + const response = await core.eval.startBatchEvaluation( + { + name: values.name, + evaluatorIds: values.evaluatorIds, + source: { origin: "agent", agent: values.agent, window: { startTime, endTime } }, + }, + opts, + ); + setResult(response); + return response; + }} + runningLabel={`starting batch evaluation ${values.name}…`} + successLabel={ + result + ? `batch evaluation '${result.batchEvaluationName}' started (${result.status})` + : "batch evaluation started" + } + successNextSteps={ + result?.batchEvaluationId + ? [`agentcore eval batch-evaluation get ${result.batchEvaluationId}`] + : undefined + } + onDone={() => navigate(MENU)} + doneLabel="go back" + > + + set({ name })} + required + schema={BatchEvaluationNameSchema} + live + /> + + + + set({ agent })} + required + /> + + + + set({ lookbackDays })} + required + schema={LookbackSchema} + live + /> + + + + set({ evaluatorIds })} + /> + + + + + + + ); +} + +function summaryOf(values: EvaluateFormValues): Record { + return { + name: values.name, + source: `agent · ${values.agent}`, + lookback: `last ${values.lookbackDays} days`, + evaluators: values.evaluatorIds.join(", "), + }; +} + +// EvaluatorMultiSelect is a compound wizard field: the shell ships no +// multi-select, so this owns its own useInput — up/down move the cursor, space +// toggles, enter advances once at least one evaluator is checked. +function EvaluatorMultiSelect({ + evaluators, + loading, + error, + selected, + onChange, +}: { + evaluators: EvaluatorSummary[]; + loading: boolean; + error?: Error; + selected: string[]; + onChange: (ids: string[]) => void; +}) { + const { advance, back } = useWizard(); + const [cursor, setCursor] = useState(0); + const [issue, setIssue] = useState(); + + useKeyHints([ + { key: "↑↓", label: "navigate" }, + { key: "space", label: "toggle" }, + { key: "enter", label: "continue" }, + ]); + + const withId = evaluators.filter((evaluator) => evaluator.evaluatorId); + // Clamp on read rather than in the arrow setters so the handler never depends + // on a stale list length (the query resolves after the field first mounts). + const active = withId.length === 0 ? 0 : Math.min(Math.max(cursor, 0), withId.length - 1); + + useInput((input, key) => { + if (key.escape) { + back(); + return; + } + if (key.upArrow) { + setCursor(Math.max(0, active - 1)); + return; + } + if (key.downArrow) { + setCursor(Math.min(withId.length - 1, active + 1)); + return; + } + if (input === " ") { + const id = withId[active]?.evaluatorId; + if (!id) return; + onChange(selected.includes(id) ? selected.filter((s) => s !== id) : [...selected, id]); + setIssue(undefined); + return; + } + if (key.return) { + if (selected.length === 0) { + setIssue("select at least one evaluator"); + return; + } + advance(); + } + }); + + if (loading) return loading evaluators…; + if (error) return {error.message}; + if (withId.length === 0) { + return no evaluators found in this Region; + } + + // Window the list so a long account roster doesn't push the footer off-screen; + // the cursor stays centred until it reaches either end. + const WINDOW = 8; + const start = Math.max(0, Math.min(active - Math.floor(WINDOW / 2), withId.length - WINDOW)); + const visible = withId.slice(start, start + WINDOW); + + return ( + + {visible.map((evaluator, i) => { + const id = evaluator.evaluatorId!; + const checked = selected.includes(id); + const focused = start + i === active; + return ( + + {focused ? glyphs.pointer : " "} [{checked ? glyphs.check : " "}]{" "} + {evaluator.evaluatorName ?? id} + + ); + })} + + {issue ? ( + {issue} + ) : ( + + {selected.length} selected · {active + 1}/{withId.length} + + )} + + + ); +} diff --git a/src/handlers/eval/batch-evaluation/index.tsx b/src/handlers/eval/batch-evaluation/index.tsx index 1fe9c9aa5..aa2a9e13d 100644 --- a/src/handlers/eval/batch-evaluation/index.tsx +++ b/src/handlers/eval/batch-evaluation/index.tsx @@ -9,13 +9,13 @@ import { createEvaluateBatchEvaluationHandler } from "./evaluate"; import { createSimulateBatchEvaluationHandler } from "./simulate"; // batch-evaluation supports evaluate + simulate (start jobs) plus get + list. A -// bare invocation opens the interactive TUI (list → get), matching evaluator and -// online-eval; evaluate/simulate appear below the command-line-only divider. +// bare invocation opens the interactive TUI menu; get, list, and evaluate each +// have a screen, while simulate stays command-line-only (below the divider). export function createBatchEvaluationHandler(core: Core, io: AppIO): Router { return new Router("batch-evaluation", "run and inspect AgentCore batch evaluations") .use(withTuiOnEmptyFlagsAndArgs(core, io)) .default(renderTui(core, io)) - .supportedTuiCommands("get", "list") + .supportedTuiCommands("get", "list", "evaluate") .handler(createEvaluateBatchEvaluationHandler(core, io)) .handler(createSimulateBatchEvaluationHandler(core, io)) .handler(createGetBatchEvaluationHandler(core, io)) diff --git a/src/projectSchemas/batch-evaluation.test.ts b/src/projectSchemas/batch-evaluation.test.ts new file mode 100644 index 000000000..f7f7d319c --- /dev/null +++ b/src/projectSchemas/batch-evaluation.test.ts @@ -0,0 +1,12 @@ +import { test, expect } from "bun:test"; +import { BatchEvaluationNameSchema } from "./batch-evaluation"; + +test("BatchEvaluationNameSchema accepts letters, digits and underscores", () => { + expect(BatchEvaluationNameSchema.safeParse("nightly_regression_1").success).toBe(true); +}); + +test("BatchEvaluationNameSchema rejects hyphens, a leading digit, and empty", () => { + expect(BatchEvaluationNameSchema.safeParse("nightly-regression").success).toBe(false); + expect(BatchEvaluationNameSchema.safeParse("1nightly").success).toBe(false); + expect(BatchEvaluationNameSchema.safeParse("").success).toBe(false); +}); diff --git a/src/projectSchemas/batch-evaluation.ts b/src/projectSchemas/batch-evaluation.ts new file mode 100644 index 000000000..08e9af943 --- /dev/null +++ b/src/projectSchemas/batch-evaluation.ts @@ -0,0 +1,12 @@ +import { z } from "zod"; + +// The service constrains batchEvaluationName to this pattern; mirrors the sibling +// eval resource name schemas (evaluator, online-eval-config). +export const BatchEvaluationNameSchema = z + .string() + .min(1, "Name is required") + .max(48) + .regex( + /^[a-zA-Z][a-zA-Z0-9_]{0,47}$/, + "Must begin with a letter and contain only alphanumeric characters and underscores (max 48 chars)", + );