feat(online-eval): add --output-config and --tags, --description on update

src/core/eval.tsx

@@ -980,6 +980,9 @@ export class EvalClient implements CoreEvalClient {
           options.region,
           logGroupNamesOf(dataSourceConfig),
           await evaluatorKmsKeys(input.evaluatorIds ?? [], control),
+          // Read only to widen the write scope to the chosen destination; the
+          // request object below still gets the caller's object untouched.
+          { outputConfig: input.outputConfig },
         )
       ).roleArn;
 
@@ -989,8 +992,10 @@ export class EvalClient implements CoreEvalClient {
       rule: toRule(input.samplingRate, input.sessionTimeoutMinutes, input.filters),
       dataSourceConfig,
       evaluators: input.evaluatorIds?.map((evaluatorId) => ({ evaluatorId })),
+      outputConfig: input.outputConfig,
       evaluationExecutionRoleArn,
       enableOnCreate: input.enableOnCreate ?? true,
+      tags: input.tags,
     });
 
     // A role provisioned moments ago may not be assumable yet (IAM is eventually
@@ -1220,9 +1225,11 @@ export class EvalClient implements CoreEvalClient {
     const response = await control.send(
       new UpdateOnlineEvaluationConfigCommand({
         onlineEvaluationConfigId: id,
+        description: update.description,
         rule: toRule(samplingPercentage, sessionTimeoutMinutes, filters),
         dataSourceConfig,
         evaluators,
+        outputConfig: update.outputConfig,
         evaluationExecutionRoleArn: update.evaluationExecutionRoleArn,
       }),
     );

src/core/onlineEvalExecutionRole.test.ts

@@ -110,3 +110,68 @@ test("gives identical policies the same name", () => {
     scopePolicyName(executionPolicy(REGION, ACCOUNT, ["/a*", "/b*"], [])),
   );
 });
+
+function writeStatement(policy: string) {
+  return statements(policy).find((s) => s.Sid === "WriteEvaluationResults");
+}
+
+const SERVICE_RESULTS = `arn:aws:logs:${REGION}:${ACCOUNT}:log-group:/aws/bedrock-agentcore/evaluations/*`;
+
+test("a config with no output destination keeps the service namespace as a bare string", () => {
+  const write = writeStatement(executionPolicy(REGION, ACCOUNT, LOG_GROUPS, []));
+
+  expect(write?.Resource).toBe(SERVICE_RESULTS);
+  expect(Array.isArray(write?.Resource)).toBe(false);
+});
+
+test("a customer-named dedicated group is granted alongside the service namespace", () => {
+  const write = writeStatement(
+    executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [], {
+      cloudWatchConfig: {
+        logGroupName: "/company/agent-evaluations",
+        resultDestination: "DEDICATED_LOG_GROUP",
+      },
+    }),
+  );
+
+  expect(write?.Resource).toEqual([
+    SERVICE_RESULTS,
+    `arn:aws:logs:${REGION}:${ACCOUNT}:log-group:/company/agent-evaluations*`,
+  ]);
+  expect(write?.Action).toContain("logs:CreateLogGroup");
+});
+
+test("SOURCE_LOG_GROUP grants writes to the groups the traces are read from", () => {
+  const write = writeStatement(
+    executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [], {
+      cloudWatchConfig: { resultDestination: "SOURCE_LOG_GROUP" },
+    }),
+  );
+
+  expect(write?.Resource).toEqual([
+    SERVICE_RESULTS,
+    `arn:aws:logs:${REGION}:${ACCOUNT}:log-group:/aws/bedrock-agentcore/runtimes/orders-agent-abc123*`,
+  ]);
+});
+
+test("a destination already inside the service namespace adds nothing", () => {
+  const write = writeStatement(
+    executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [], {
+      cloudWatchConfig: {
+        logGroupName: "/aws/bedrock-agentcore/evaluations/online-evaluations/results/default",
+        resultDestination: "DEDICATED_LOG_GROUP",
+      },
+    }),
+  );
+
+  expect(write?.Resource).toBe(SERVICE_RESULTS);
+});
+
+test("changing the destination changes the policy name, so a re-scope is a new grant", () => {
+  const before = executionPolicy(REGION, ACCOUNT, LOG_GROUPS, []);
+  const after = executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [], {
+    cloudWatchConfig: { logGroupName: "/company/agent-evaluations" },
+  });
+
+  expect(scopePolicyName(after)).not.toBe(scopePolicyName(before));
+});

src/core/onlineEvalExecutionRole.tsx

@@ -15,7 +15,6 @@ import { parseArn, resourceNameFromArn } from "./arn";
 // evaluation results back to CloudWatch. When the caller doesn't bring one,
 // OnlineEvalClient provisions a per-config default here, scoped to the log
 // group(s) being sampled. Idempotent: an existing role is reused.
-//
 // Each scope is stored as its own inline policy, named after a fingerprint of the
 // scope, so granting a new scope never overwrites the policy backing the current
 // one. IAM unions Allows across a role's inline policies, which lets an update
@@ -83,10 +82,35 @@ function runtimeLogGroupPrefix(logGroupName: string): string {
   return match?.[1] ?? logGroupName;
 }
 
+const SERVICE_RESULT_PREFIX = "/aws/bedrock-agentcore/evaluations/";
+
+function resultWriteArns(
+  logs: string,
+  sampledArns: string[],
+  outputConfig: OnlineEvalResultDestination | undefined,
+): string | string[] {
+  const arns = [`${logs}:${SERVICE_RESULT_PREFIX}*`];
+  const cloudWatch = outputConfig?.cloudWatchConfig;
+
+  if (cloudWatch?.resultDestination === "SOURCE_LOG_GROUP") {
+    arns.push(...sampledArns);
+  } else if (
+    cloudWatch?.logGroupName &&
+    !cloudWatch.logGroupName.startsWith(SERVICE_RESULT_PREFIX)
+  ) {
+    arns.push(`${logs}:${cloudWatch.logGroupName}*`);
+  }
+
+  return arns.length === 1 ? arns[0]! : arns;
+}
+
+export type OnlineEvalResultDestination = {
+  cloudWatchConfig?: { logGroupName?: string; resultDestination?: string } | undefined;
+};
+
 // executionPolicy grants the permissions CreateOnlineEvaluationConfig validates
 // at creation time. Exported for assertion: the policy body is not observable
 // through the recorded IAM fixtures, whose responses are empty.
-//
 // at creation time: Logs Insights query access over the sampled log groups plus
 // the `aws/spans` group that carries the actual trace spans, Bedrock model
 // invocation for LLM-as-a-Judge evaluators, Lambda invocation for code-based
@@ -98,6 +122,7 @@ export function executionPolicy(
   accountId: string,
   logGroupNames: string[],
   kmsKeyArns: string[],
+  outputConfig?: OnlineEvalResultDestination,
 ): string {
   const logs = `arn:aws:logs:${region}:${accountId}:log-group`;
   const spansArn = `${logs}:aws/spans`;
@@ -134,6 +159,8 @@ export function executionPolicy(
         Resource: [`${spansArn}*`, ...sampledArns],
       },
       {
+        // logs:CreateLogGroup is needed because the service creates a
+        // customer-named result group that does not exist yet.
         Sid: "WriteEvaluationResults",
         Effect: "Allow",
         Action: [
@@ -142,7 +169,7 @@ export function executionPolicy(
           "logs:DescribeLogStreams",
           "logs:PutLogEvents",
         ],
-        Resource: `${logs}:/aws/bedrock-agentcore/evaluations/*`,
+        Resource: resultWriteArns(logs, sampledArns, outputConfig),
       },
       {
         Sid: "IndexSpans",
@@ -201,6 +228,11 @@ export function scopePolicyName(policyDocument: string): string {
   return `${POLICY_PREFIX}-${fingerprint(policyDocument)}`;
 }
 
+export type GrantScopeOptions = {
+  roleName?: string;
+  outputConfig?: OnlineEvalResultDestination;
+};
+
 // grantOnlineEvalScope creates the execution role for `configName` if it does not
 // exist and attaches the inline policy for this scope, returning the role ARN and
 // the policy name written. The caller revokes the superseded scope once whatever
@@ -211,7 +243,7 @@ export async function grantOnlineEvalScope(
   region: string,
   logGroupNames: string[],
   kmsKeyArns: string[] = [],
-  roleName = onlineEvalExecutionRoleName(configName),
+  { roleName = onlineEvalExecutionRoleName(configName), outputConfig }: GrantScopeOptions = {},
 ): Promise<{ roleArn: string; policyName: string }> {
   let roleArn: string;
   try {
@@ -234,6 +266,7 @@ export async function grantOnlineEvalScope(
     accountIdFromRoleArn(roleArn),
     logGroupNames,
     kmsKeyArns,
+    outputConfig,
   );
   const policyName = scopePolicyName(policyDocument);
   await iam.send(

src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.5e70e8cf174a7bc6.json

@@ -1,14 +1,14 @@
 {
-  "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_role_warn-2KNTGuGVDl",
-  "onlineEvaluationConfigId": "agentcore_cli_online_eval_role_warn-2KNTGuGVDl",
+  "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_role_warn-tSVCZiBf4l",
+  "onlineEvaluationConfigId": "agentcore_cli_online_eval_role_warn-tSVCZiBf4l",
   "createdAt": {
-    "$date": "2026-08-03T21:29:57.112Z"
+    "$date": "2026-09-15T15:11:36.526Z"
   },
   "status": "CREATING",
   "executionStatus": "DISABLED",
   "outputConfig": {
     "cloudWatchConfig": {
-      "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_role_warn-2KNTGuGVDl"
+      "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_role_warn-tSVCZiBf4l"
     }
   }
 }

src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.6b79e5be866896d6.json

@@ -1,14 +1,14 @@
 {
-  "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_kms-bG4DUW3Ua5",
-  "onlineEvaluationConfigId": "agentcore_cli_online_eval_kms-bG4DUW3Ua5",
+  "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_kms-15ciiv2UoV",
+  "onlineEvaluationConfigId": "agentcore_cli_online_eval_kms-15ciiv2UoV",
   "createdAt": {
-    "$date": "2026-08-03T21:29:51.133Z"
+    "$date": "2026-09-15T15:11:30.185Z"
   },
   "status": "CREATING",
   "executionStatus": "DISABLED",
   "outputConfig": {
     "cloudWatchConfig": {
-      "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_kms-bG4DUW3Ua5"
+      "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_kms-15ciiv2UoV"
     }
   }
 }

src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.bdca68c635390f67.json → src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.f9d2b8ebfad2219b.json

@@ -1,14 +1,12 @@
 {
-  "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk",
-  "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk",
+  "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-5gZw987afd",
+  "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-5gZw987afd",
   "createdAt": {
-    "$date": "2026-08-03T21:29:43.069Z"
+    "$date": "2026-09-15T15:11:21.000Z"
   },
   "status": "CREATING",
   "executionStatus": "DISABLED",
   "outputConfig": {
-    "cloudWatchConfig": {
-      "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_fixture-vYkaD93sFk"
-    }
+    "cloudWatchConfig": {}
   }
 }

src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.fa01de9ac60dc685.json → src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.2ae2ccb0272b469e.json

@@ -1,5 +1,5 @@
 {
-  "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_role_warn-2KNTGuGVDl",
-  "onlineEvaluationConfigId": "agentcore_cli_online_eval_role_warn-2KNTGuGVDl",
+  "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_role_warn-tSVCZiBf4l",
+  "onlineEvaluationConfigId": "agentcore_cli_online_eval_role_warn-tSVCZiBf4l",
   "status": "DELETING"
 }

src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.77430112d0720f6.json → src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.d1856366f3636123.json

@@ -1,5 +1,5 @@
 {
-  "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_kms-bG4DUW3Ua5",
-  "onlineEvaluationConfigId": "agentcore_cli_online_eval_kms-bG4DUW3Ua5",
+  "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_kms-15ciiv2UoV",
+  "onlineEvaluationConfigId": "agentcore_cli_online_eval_kms-15ciiv2UoV",
   "status": "DELETING"
 }

src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.a952ff6787c06557.json → src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.e3250f36c85f44ba.json

@@ -1,5 +1,5 @@
 {
-  "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk",
-  "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk",
+  "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-5gZw987afd",
+  "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-5gZw987afd",
   "status": "DELETING"
 }

src/handlers/eval/online-eval/__fixtures__/GetAgentRuntimeCommand.80bb567edcaa4bb1.json

@@ -10,14 +10,14 @@
     "$date": "2026-03-24T16:13:08.198Z"
   },
   "roleArn": "arn:aws:iam::725476964917:role/AgentCore-myimport-defaul-ApplicationAgentTestAgent-tJFjyd6jLIOn",
-  "networkConfiguration": {
-    "networkMode": "PUBLIC"
-  },
   "status": "READY",
   "lifecycleConfiguration": {
     "idleRuntimeSessionTimeout": 900,
     "maxLifetime": 28800
   },
+  "networkConfiguration": {
+    "networkMode": "PUBLIC"
+  },
   "description": "AgentCore Runtime: myimport_testAgent_Agent",
   "workloadIdentityDetails": {
     "workloadIdentityArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:workload-identity-directory/default/workload-identity/testAgent_Agent-wm9hYBD93Y"

src/handlers/eval/online-eval/__fixtures__/GetAgentRuntimeCommand.d729967662bebc9f.json

@@ -10,14 +10,14 @@
     "$date": "2026-06-17T22:09:21.093Z"
   },
   "roleArn": "arn:aws:iam::725476964917:role/AgentCore-ABVfyLatest-def-ApplicationAgentABVfyLate-b6b570G88FJ3",
-  "networkConfiguration": {
-    "networkMode": "PUBLIC"
-  },
   "status": "READY",
   "lifecycleConfiguration": {
     "idleRuntimeSessionTimeout": 900,
     "maxLifetime": 28800
   },
+  "networkConfiguration": {
+    "networkMode": "PUBLIC"
+  },
   "description": "AgentCore Runtime: ABVfyLatest_ABVfyLatest",
   "workloadIdentityDetails": {
     "workloadIdentityArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:workload-identity-directory/default/workload-identity/ABVfyLatest_ABVfyLatest-PFLr353QVA"

src/handlers/eval/online-eval/__fixtures__/GetEvaluatorCommand.5ee939ef1fda1ca7.json

@@ -2,6 +2,26 @@
   "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness",
   "evaluatorId": "Builtin.Correctness",
   "evaluatorName": "Builtin.Correctness",
+  "evaluatorConfig": {
+    "llmAsAJudge": {
+      "ratingScale": {
+        "numerical": [
+          {
+            "value": 0,
+            "label": "Incorrect"
+          },
+          {
+            "value": 0.5,
+            "label": "Partially correct"
+          },
+          {
+            "value": 1,
+            "label": "Correct"
+          }
+        ]
+      }
+    }
+  },
   "level": "TRACE",
   "status": "ACTIVE",
   "createdAt": {
@@ -10,5 +30,8 @@
   "updatedAt": {
     "$date": "2024-10-22T00:00:00.000Z"
   },
-  "kmsKeyArn": "arn:aws:kms:us-west-2:725476964917:key/31a2dd2f-c8a0-42b8-9f52-12e5ecb22468"
-}
+  "description": "Response Quality Metric. Evaluates whether the information in the agent's response is factually accurate",
+  "evaluatorType": "Builtin",
+  "provider": "AWS",
+  "lockedForModification": true
+}

src/handlers/eval/online-eval/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json

@@ -47,5 +47,7 @@
     "$date": "2024-10-22T00:00:00.000Z"
   },
   "description": "Response Quality Metric. Evaluates from user's perspective how useful and valuable the agent's response is",
+  "evaluatorType": "Builtin",
+  "provider": "AWS",
   "lockedForModification": true
 }