diff --git a/src/components/CliOnlyScreen.test.tsx b/src/components/CliOnlyScreen.test.tsx
index dce691c2b..6116dbcb9 100644
--- a/src/components/CliOnlyScreen.test.tsx
+++ b/src/components/CliOnlyScreen.test.tsx
@@ -170,16 +170,15 @@ describe("option help groups", () => {
const headingLine = (title: string) => `\n ${title}\n`;
test("a grouped command renders one section per heading, in --help order", async () => {
- const r = renderScreen("/agentcore/eval/batch-evaluation/evaluate");
+ // simulate is a grouped, command-line-only sibling of evaluate (which now
+ // has a TUI screen), so it exercises the CliOnlyScreen help-group rendering.
+ const r = renderScreen("/agentcore/eval/batch-evaluation/simulate");
await waitForText(r.lastFrame, "this command runs from the command line");
const frame = r.lastFrame()!;
- const positions = [
- "configuration",
- "session source (choose exactly one)",
- "source filters",
- "evaluation",
- ].map((title) => frame.indexOf(headingLine(title)));
+ const positions = ["runtime invocation", "dataset", "configuration", "evaluation"].map(
+ (title) => frame.indexOf(headingLine(title)),
+ );
expect(positions.every((position) => position >= 0)).toBe(true);
expect(positions).toEqual([...positions].sort((a, b) => a - b));
diff --git a/src/components/Root.tsx b/src/components/Root.tsx
index bbbf23c5b..8cb1ac1f3 100644
--- a/src/components/Root.tsx
+++ b/src/components/Root.tsx
@@ -58,6 +58,7 @@ import {
import { BatchEvaluationScreen } from "../handlers/eval/batch-evaluation/screen.tsx";
import { BatchEvaluationListScreen } from "../handlers/eval/batch-evaluation/list/screen.tsx";
import { BatchEvaluationGetJsonScreen } from "../handlers/eval/batch-evaluation/get/screen.tsx";
+import { BatchEvaluationEvaluateScreen } from "../handlers/eval/batch-evaluation/evaluate/screen.tsx";
import { RecommendationScreen } from "../handlers/eval/recommendation/screen.tsx";
import { RecommendationListScreen } from "../handlers/eval/recommendation/list/screen.tsx";
import { RecommendationGetJsonScreen } from "../handlers/eval/recommendation/get/screen.tsx";
@@ -612,6 +613,10 @@ export function Root({ path, ctx, core, queryClient }: RootProps) {
path="agentcore/eval/batch-evaluation/list"
element={}
/>
+ }
+ />
{/* Bare `get` (no id) has nothing to show — send the user to the list. */}
({
+ name: "",
+ agent: "",
+ lookbackDays: "7",
+ evaluatorIds: [],
+ });
+ const [result, setResult] = useState();
+ const set = (update: Partial) =>
+ setValues((current) => ({ ...current, ...update }));
+
+ // First page only for now; a paged picker is a follow-up (see design doc).
+ // ponytail: 100-item cap, add pagination if accounts routinely exceed it.
+ const evaluators = useQuery({
+ queryKey: ["evaluators", opts.region],
+ queryFn: () => core.eval.listEvaluators(undefined, 100, opts),
+ });
+
+ return (
+ navigate(MENU)}
+ onSubmit={async () => {
+ const endTime = new Date();
+ const startTime = new Date(endTime.getTime() - Number(values.lookbackDays) * 86_400_000);
+ const response = await core.eval.startBatchEvaluation(
+ {
+ name: values.name,
+ evaluatorIds: values.evaluatorIds,
+ source: { origin: "agent", agent: values.agent, window: { startTime, endTime } },
+ },
+ opts,
+ );
+ setResult(response);
+ return response;
+ }}
+ runningLabel={`starting batch evaluation ${values.name}…`}
+ successLabel={
+ result
+ ? `batch evaluation '${result.batchEvaluationName}' started (${result.status})`
+ : "batch evaluation started"
+ }
+ successNextSteps={
+ result?.batchEvaluationId
+ ? [`agentcore eval batch-evaluation get ${result.batchEvaluationId}`]
+ : undefined
+ }
+ onDone={() => navigate(MENU)}
+ doneLabel="go back"
+ >
+
+ set({ name })}
+ required
+ schema={BatchEvaluationNameSchema}
+ live
+ />
+
+
+
+ set({ agent })}
+ required
+ />
+
+
+
+ set({ lookbackDays })}
+ required
+ schema={LookbackSchema}
+ live
+ />
+
+
+
+ set({ evaluatorIds })}
+ />
+
+
+
+
+
+
+ );
+}
+
+function summaryOf(values: EvaluateFormValues): Record {
+ return {
+ name: values.name,
+ source: `agent · ${values.agent}`,
+ lookback: `last ${values.lookbackDays} days`,
+ evaluators: values.evaluatorIds.join(", "),
+ };
+}
+
+// EvaluatorMultiSelect is a compound wizard field: the shell ships no
+// multi-select, so this owns its own useInput — up/down move the cursor, space
+// toggles, enter advances once at least one evaluator is checked.
+function EvaluatorMultiSelect({
+ evaluators,
+ loading,
+ error,
+ selected,
+ onChange,
+}: {
+ evaluators: EvaluatorSummary[];
+ loading: boolean;
+ error?: Error;
+ selected: string[];
+ onChange: (ids: string[]) => void;
+}) {
+ const { advance, back } = useWizard();
+ const [cursor, setCursor] = useState(0);
+ const [issue, setIssue] = useState();
+
+ useKeyHints([
+ { key: "↑↓", label: "navigate" },
+ { key: "space", label: "toggle" },
+ { key: "enter", label: "continue" },
+ ]);
+
+ const withId = evaluators.filter((evaluator) => evaluator.evaluatorId);
+ // Clamp on read rather than in the arrow setters so the handler never depends
+ // on a stale list length (the query resolves after the field first mounts).
+ const active = withId.length === 0 ? 0 : Math.min(Math.max(cursor, 0), withId.length - 1);
+
+ useInput((input, key) => {
+ if (key.escape) {
+ back();
+ return;
+ }
+ if (key.upArrow) {
+ setCursor(Math.max(0, active - 1));
+ return;
+ }
+ if (key.downArrow) {
+ setCursor(Math.min(withId.length - 1, active + 1));
+ return;
+ }
+ if (input === " ") {
+ const id = withId[active]?.evaluatorId;
+ if (!id) return;
+ onChange(selected.includes(id) ? selected.filter((s) => s !== id) : [...selected, id]);
+ setIssue(undefined);
+ return;
+ }
+ if (key.return) {
+ if (selected.length === 0) {
+ setIssue("select at least one evaluator");
+ return;
+ }
+ advance();
+ }
+ });
+
+ if (loading) return loading evaluators…;
+ if (error) return {error.message};
+ if (withId.length === 0) {
+ return no evaluators found in this Region;
+ }
+
+ // Window the list so a long account roster doesn't push the footer off-screen;
+ // the cursor stays centred until it reaches either end.
+ const WINDOW = 8;
+ const start = Math.max(0, Math.min(active - Math.floor(WINDOW / 2), withId.length - WINDOW));
+ const visible = withId.slice(start, start + WINDOW);
+
+ return (
+
+ {visible.map((evaluator, i) => {
+ const id = evaluator.evaluatorId!;
+ const checked = selected.includes(id);
+ const focused = start + i === active;
+ return (
+
+ {focused ? glyphs.pointer : " "} [{checked ? glyphs.check : " "}]{" "}
+ {evaluator.evaluatorName ?? id}
+
+ );
+ })}
+
+ {issue ? (
+ {issue}
+ ) : (
+
+ {selected.length} selected · {active + 1}/{withId.length}
+
+ )}
+
+
+ );
+}
diff --git a/src/handlers/eval/batch-evaluation/index.tsx b/src/handlers/eval/batch-evaluation/index.tsx
index 1fe9c9aa5..aa2a9e13d 100644
--- a/src/handlers/eval/batch-evaluation/index.tsx
+++ b/src/handlers/eval/batch-evaluation/index.tsx
@@ -9,13 +9,13 @@ import { createEvaluateBatchEvaluationHandler } from "./evaluate";
import { createSimulateBatchEvaluationHandler } from "./simulate";
// batch-evaluation supports evaluate + simulate (start jobs) plus get + list. A
-// bare invocation opens the interactive TUI (list → get), matching evaluator and
-// online-eval; evaluate/simulate appear below the command-line-only divider.
+// bare invocation opens the interactive TUI menu; get, list, and evaluate each
+// have a screen, while simulate stays command-line-only (below the divider).
export function createBatchEvaluationHandler(core: Core, io: AppIO): Router {
return new Router("batch-evaluation", "run and inspect AgentCore batch evaluations")
.use(withTuiOnEmptyFlagsAndArgs(core, io))
.default(renderTui(core, io))
- .supportedTuiCommands("get", "list")
+ .supportedTuiCommands("get", "list", "evaluate")
.handler(createEvaluateBatchEvaluationHandler(core, io))
.handler(createSimulateBatchEvaluationHandler(core, io))
.handler(createGetBatchEvaluationHandler(core, io))
diff --git a/src/projectSchemas/batch-evaluation.test.ts b/src/projectSchemas/batch-evaluation.test.ts
new file mode 100644
index 000000000..f7f7d319c
--- /dev/null
+++ b/src/projectSchemas/batch-evaluation.test.ts
@@ -0,0 +1,12 @@
+import { test, expect } from "bun:test";
+import { BatchEvaluationNameSchema } from "./batch-evaluation";
+
+test("BatchEvaluationNameSchema accepts letters, digits and underscores", () => {
+ expect(BatchEvaluationNameSchema.safeParse("nightly_regression_1").success).toBe(true);
+});
+
+test("BatchEvaluationNameSchema rejects hyphens, a leading digit, and empty", () => {
+ expect(BatchEvaluationNameSchema.safeParse("nightly-regression").success).toBe(false);
+ expect(BatchEvaluationNameSchema.safeParse("1nightly").success).toBe(false);
+ expect(BatchEvaluationNameSchema.safeParse("").success).toBe(false);
+});
diff --git a/src/projectSchemas/batch-evaluation.ts b/src/projectSchemas/batch-evaluation.ts
new file mode 100644
index 000000000..08e9af943
--- /dev/null
+++ b/src/projectSchemas/batch-evaluation.ts
@@ -0,0 +1,12 @@
+import { z } from "zod";
+
+// The service constrains batchEvaluationName to this pattern; mirrors the sibling
+// eval resource name schemas (evaluator, online-eval-config).
+export const BatchEvaluationNameSchema = z
+ .string()
+ .min(1, "Name is required")
+ .max(48)
+ .regex(
+ /^[a-zA-Z][a-zA-Z0-9_]{0,47}$/,
+ "Must begin with a letter and contain only alphanumeric characters and underscores (max 48 chars)",
+ );