feat(eval): interactive TUI wizard for batch-evaluation evaluate

src/components/CliOnlyScreen.test.tsx

@@ -170,16 +170,15 @@ describe("option help groups", () => {
   const headingLine = (title: string) => `\n ${title}\n`;
 
   test("a grouped command renders one section per heading, in --help order", async () => {
-    const r = renderScreen("/agentcore/eval/batch-evaluation/evaluate");
+    // simulate is a grouped, command-line-only sibling of evaluate (which now
+    // has a TUI screen), so it exercises the CliOnlyScreen help-group rendering.
+    const r = renderScreen("/agentcore/eval/batch-evaluation/simulate");
 
     await waitForText(r.lastFrame, "this command runs from the command line");
     const frame = r.lastFrame()!;
-    const positions = [
-      "configuration",
-      "session source (choose exactly one)",
-      "source filters",
-      "evaluation",
-    ].map((title) => frame.indexOf(headingLine(title)));
+    const positions = ["runtime invocation", "dataset", "configuration", "evaluation"].map(
+      (title) => frame.indexOf(headingLine(title)),
+    );
 
     expect(positions.every((position) => position >= 0)).toBe(true);
     expect(positions).toEqual([...positions].sort((a, b) => a - b));

src/components/Root.tsx

@@ -58,6 +58,7 @@ import {
 import { BatchEvaluationScreen } from "../handlers/eval/batch-evaluation/screen.tsx";
 import { BatchEvaluationListScreen } from "../handlers/eval/batch-evaluation/list/screen.tsx";
 import { BatchEvaluationGetJsonScreen } from "../handlers/eval/batch-evaluation/get/screen.tsx";
+import { BatchEvaluationEvaluateScreen } from "../handlers/eval/batch-evaluation/evaluate/screen.tsx";
 import { RecommendationScreen } from "../handlers/eval/recommendation/screen.tsx";
 import { RecommendationListScreen } from "../handlers/eval/recommendation/list/screen.tsx";
 import { RecommendationGetJsonScreen } from "../handlers/eval/recommendation/get/screen.tsx";
@@ -612,6 +613,10 @@ export function Root({ path, ctx, core, queryClient }: RootProps) {
             path="agentcore/eval/batch-evaluation/list"
             element={<BatchEvaluationListScreen ctx={ctx} core={core} />}
           />
+          <Route
+            path="agentcore/eval/batch-evaluation/evaluate"
+            element={<BatchEvaluationEvaluateScreen ctx={ctx} core={core} />}
+          />
           {/* Bare `get` (no id) has nothing to show — send the user to the list. */}
           <Route
             path="agentcore/eval/batch-evaluation/get"

src/handlers/eval/batch-evaluation/evaluate/screen.tsx

@@ -0,0 +1,246 @@
+import { useState } from "react";
+import { Box, Text, useInput } from "ink";
+import { useNavigate } from "react-router";
+import { useQuery } from "@tanstack/react-query";
+import z from "zod";
+import type { StartBatchEvaluationResponse } from "@aws-sdk/client-bedrock-agentcore";
+import type { EvaluatorSummary } from "@aws-sdk/client-bedrock-agentcore-control";
+import type { ScreenProps } from "../../../types";
+import { coreOptsFromCtx } from "../../../utils";
+import { BatchEvaluationNameSchema } from "../../../../projectSchemas/batch-evaluation";
+import {
+  Step,
+  Summary,
+  TextField,
+  Wizard,
+  useKeyHints,
+  useWizard,
+} from "../../../../components/wizard";
+import { darkTheme, glyphs } from "../../../../components/ui/_core.js";
+
+const theme = darkTheme;
+
+const BREADCRUMB = ["agentcore", "eval", "batch-evaluation", "evaluate"];
+const DESCRIPTION = "evaluate existing sessions service-side";
+const MENU = "/agentcore/eval/batch-evaluation";
+
+const LookbackSchema = z.coerce.number().int().positive();
+
+interface EvaluateFormValues {
+  name: string;
+  agent: string;
+  lookbackDays: string;
+  evaluatorIds: string[];
+}
+
+export function BatchEvaluationEvaluateScreen({ ctx, core }: ScreenProps) {
+  const navigate = useNavigate();
+  const opts = coreOptsFromCtx(ctx);
+  const [values, setValues] = useState<EvaluateFormValues>({
+    name: "",
+    agent: "",
+    lookbackDays: "7",
+    evaluatorIds: [],
+  });
+  const [result, setResult] = useState<StartBatchEvaluationResponse>();
+  const set = (update: Partial<EvaluateFormValues>) =>
+    setValues((current) => ({ ...current, ...update }));
+
+  // First page only for now; a paged picker is a follow-up (see design doc).
+  // ponytail: 100-item cap, add pagination if accounts routinely exceed it.
+  const evaluators = useQuery({
+    queryKey: ["evaluators", opts.region],
+    queryFn: () => core.eval.listEvaluators(undefined, 100, opts),
+  });
+
+  return (
+    <Wizard
+      breadcrumb={BREADCRUMB}
+      description={DESCRIPTION}
+      onCancel={() => navigate(MENU)}
+      onSubmit={async () => {
+        const endTime = new Date();
+        const startTime = new Date(endTime.getTime() - Number(values.lookbackDays) * 86_400_000);
+        const response = await core.eval.startBatchEvaluation(
+          {
+            name: values.name,
+            evaluatorIds: values.evaluatorIds,
+            source: { origin: "agent", agent: values.agent, window: { startTime, endTime } },
+          },
+          opts,
+        );
+        setResult(response);
+        return response;
+      }}
+      runningLabel={`starting batch evaluation ${values.name}…`}
+      successLabel={
+        result
+          ? `batch evaluation '${result.batchEvaluationName}' started (${result.status})`
+          : "batch evaluation started"
+      }
+      successNextSteps={
+        result?.batchEvaluationId
+          ? [`agentcore eval batch-evaluation get ${result.batchEvaluationId}`]
+          : undefined
+      }
+      onDone={() => navigate(MENU)}
+      doneLabel="go back"
+    >
+      <Step stepKey="name" prompt="name your batch evaluation">
+        <TextField
+          label="Name"
+          help="letters, digits and underscores, starting with a letter (max 48)"
+          placeholder="nightly_regression"
+          value={values.name}
+          onChange={(name) => set({ name })}
+          required
+          schema={BatchEvaluationNameSchema}
+          live
+        />
+      </Step>
+
+      <Step stepKey="source" prompt="point at the agent whose sessions to evaluate">
+        <TextField
+          label="Agent (harness or runtime ID)"
+          placeholder="my_agent"
+          value={values.agent}
+          onChange={(agent) => set({ agent })}
+          required
+        />
+      </Step>
+
+      <Step stepKey="lookback" prompt="evaluate sessions from the last N days">
+        <TextField
+          label="Lookback (days)"
+          placeholder="7"
+          value={values.lookbackDays}
+          onChange={(lookbackDays) => set({ lookbackDays })}
+          required
+          schema={LookbackSchema}
+          live
+        />
+      </Step>
+
+      <Step stepKey="evaluators" prompt="apply one or more evaluators">
+        <EvaluatorMultiSelect
+          evaluators={evaluators.data?.evaluators ?? []}
+          loading={evaluators.isPending}
+          error={evaluators.isError ? (evaluators.error as Error) : undefined}
+          selected={values.evaluatorIds}
+          onChange={(evaluatorIds) => set({ evaluatorIds })}
+        />
+      </Step>
+
+      <Step stepKey="review" prompt="this batch evaluation will be started">
+        <Summary items={summaryOf(values)} />
+      </Step>
+    </Wizard>
+  );
+}
+
+function summaryOf(values: EvaluateFormValues): Record<string, string> {
+  return {
+    name: values.name,
+    source: `agent · ${values.agent}`,
+    lookback: `last ${values.lookbackDays} days`,
+    evaluators: values.evaluatorIds.join(", "),
+  };
+}
+
+// EvaluatorMultiSelect is a compound wizard field: the shell ships no
+// multi-select, so this owns its own useInput — up/down move the cursor, space
+// toggles, enter advances once at least one evaluator is checked.
+function EvaluatorMultiSelect({
+  evaluators,
+  loading,
+  error,
+  selected,
+  onChange,
+}: {
+  evaluators: EvaluatorSummary[];
+  loading: boolean;
+  error?: Error;
+  selected: string[];
+  onChange: (ids: string[]) => void;
+}) {
+  const { advance, back } = useWizard();
+  const [cursor, setCursor] = useState(0);
+  const [issue, setIssue] = useState<string>();
+
+  useKeyHints([
+    { key: "↑↓", label: "navigate" },
+    { key: "space", label: "toggle" },
+    { key: "enter", label: "continue" },
+  ]);
+
+  const withId = evaluators.filter((evaluator) => evaluator.evaluatorId);
+  // Clamp on read rather than in the arrow setters so the handler never depends
+  // on a stale list length (the query resolves after the field first mounts).
+  const active = withId.length === 0 ? 0 : Math.min(Math.max(cursor, 0), withId.length - 1);
+
+  useInput((input, key) => {
+    if (key.escape) {
+      back();
+      return;
+    }
+    if (key.upArrow) {
+      setCursor(Math.max(0, active - 1));
+      return;
+    }
+    if (key.downArrow) {
+      setCursor(Math.min(withId.length - 1, active + 1));
+      return;
+    }
+    if (input === " ") {
+      const id = withId[active]?.evaluatorId;
+      if (!id) return;
+      onChange(selected.includes(id) ? selected.filter((s) => s !== id) : [...selected, id]);
+      setIssue(undefined);
+      return;
+    }
+    if (key.return) {
+      if (selected.length === 0) {
+        setIssue("select at least one evaluator");
+        return;
+      }
+      advance();
+    }
+  });
+
+  if (loading) return <Text color={theme.colors.muted}>loading evaluators…</Text>;
+  if (error) return <Text color={theme.colors.error}>{error.message}</Text>;
+  if (withId.length === 0) {
+    return <Text color={theme.colors.muted}>no evaluators found in this Region</Text>;
+  }
+
+  // Window the list so a long account roster doesn't push the footer off-screen;
+  // the cursor stays centred until it reaches either end.
+  const WINDOW = 8;
+  const start = Math.max(0, Math.min(active - Math.floor(WINDOW / 2), withId.length - WINDOW));
+  const visible = withId.slice(start, start + WINDOW);
+
+  return (
+    <Box flexDirection="column">
+      {visible.map((evaluator, i) => {
+        const id = evaluator.evaluatorId!;
+        const checked = selected.includes(id);
+        const focused = start + i === active;
+        return (
+          <Text key={id} color={focused ? theme.colors.focus : theme.colors.text}>
+            {focused ? glyphs.pointer : " "} [{checked ? glyphs.check : " "}]{" "}
+            {evaluator.evaluatorName ?? id}
+          </Text>
+        );
+      })}
+      <Box marginTop={1}>
+        {issue ? (
+          <Text color={theme.colors.error}>{issue}</Text>
+        ) : (
+          <Text color={theme.colors.muted}>
+            {selected.length} selected · {active + 1}/{withId.length}
+          </Text>
+        )}
+      </Box>
+    </Box>
+  );
+}

src/handlers/eval/batch-evaluation/index.tsx

@@ -9,13 +9,13 @@ import { createEvaluateBatchEvaluationHandler } from "./evaluate";
 import { createSimulateBatchEvaluationHandler } from "./simulate";
 
 // batch-evaluation supports evaluate + simulate (start jobs) plus get + list. A
-// bare invocation opens the interactive TUI (list → get), matching evaluator and
-// online-eval; evaluate/simulate appear below the command-line-only divider.
+// bare invocation opens the interactive TUI menu; get, list, and evaluate each
+// have a screen, while simulate stays command-line-only (below the divider).
 export function createBatchEvaluationHandler(core: Core, io: AppIO): Router {
   return new Router("batch-evaluation", "run and inspect AgentCore batch evaluations")
     .use(withTuiOnEmptyFlagsAndArgs(core, io))
     .default(renderTui(core, io))
-    .supportedTuiCommands("get", "list")
+    .supportedTuiCommands("get", "list", "evaluate")
     .handler(createEvaluateBatchEvaluationHandler(core, io))
     .handler(createSimulateBatchEvaluationHandler(core, io))
     .handler(createGetBatchEvaluationHandler(core, io))

src/projectSchemas/batch-evaluation.test.ts

@@ -0,0 +1,12 @@
+import { test, expect } from "bun:test";
+import { BatchEvaluationNameSchema } from "./batch-evaluation";
+
+test("BatchEvaluationNameSchema accepts letters, digits and underscores", () => {
+  expect(BatchEvaluationNameSchema.safeParse("nightly_regression_1").success).toBe(true);
+});
+
+test("BatchEvaluationNameSchema rejects hyphens, a leading digit, and empty", () => {
+  expect(BatchEvaluationNameSchema.safeParse("nightly-regression").success).toBe(false);
+  expect(BatchEvaluationNameSchema.safeParse("1nightly").success).toBe(false);
+  expect(BatchEvaluationNameSchema.safeParse("").success).toBe(false);
+});

src/projectSchemas/batch-evaluation.ts

@@ -0,0 +1,12 @@
+import { z } from "zod";
+
+// The service constrains batchEvaluationName to this pattern; mirrors the sibling
+// eval resource name schemas (evaluator, online-eval-config).
+export const BatchEvaluationNameSchema = z
+  .string()
+  .min(1, "Name is required")
+  .max(48)
+  .regex(
+    /^[a-zA-Z][a-zA-Z0-9_]{0,47}$/,
+    "Must begin with a letter and contain only alphanumeric characters and underscores (max 48 chars)",
+  );