diff --git a/src/core/eval.tsx b/src/core/eval.tsx index d22a11982..868f2de41 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -980,6 +980,9 @@ export class EvalClient implements CoreEvalClient { options.region, logGroupNamesOf(dataSourceConfig), await evaluatorKmsKeys(input.evaluatorIds ?? [], control), + // Read only to widen the write scope to the chosen destination; the + // request object below still gets the caller's object untouched. + { outputConfig: input.outputConfig }, ) ).roleArn; @@ -989,8 +992,10 @@ export class EvalClient implements CoreEvalClient { rule: toRule(input.samplingRate, input.sessionTimeoutMinutes, input.filters), dataSourceConfig, evaluators: input.evaluatorIds?.map((evaluatorId) => ({ evaluatorId })), + outputConfig: input.outputConfig, evaluationExecutionRoleArn, enableOnCreate: input.enableOnCreate ?? true, + tags: input.tags, }); // A role provisioned moments ago may not be assumable yet (IAM is eventually @@ -1220,9 +1225,11 @@ export class EvalClient implements CoreEvalClient { const response = await control.send( new UpdateOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id, + description: update.description, rule: toRule(samplingPercentage, sessionTimeoutMinutes, filters), dataSourceConfig, evaluators, + outputConfig: update.outputConfig, evaluationExecutionRoleArn: update.evaluationExecutionRoleArn, }), ); diff --git a/src/core/onlineEvalExecutionRole.test.ts b/src/core/onlineEvalExecutionRole.test.ts index 7f1ec01ea..e4a719410 100644 --- a/src/core/onlineEvalExecutionRole.test.ts +++ b/src/core/onlineEvalExecutionRole.test.ts @@ -110,3 +110,68 @@ test("gives identical policies the same name", () => { scopePolicyName(executionPolicy(REGION, ACCOUNT, ["/a*", "/b*"], [])), ); }); + +function writeStatement(policy: string) { + return statements(policy).find((s) => s.Sid === "WriteEvaluationResults"); +} + +const SERVICE_RESULTS = `arn:aws:logs:${REGION}:${ACCOUNT}:log-group:/aws/bedrock-agentcore/evaluations/*`; + +test("a config with no output destination keeps the service namespace as a bare string", () => { + const write = writeStatement(executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [])); + + expect(write?.Resource).toBe(SERVICE_RESULTS); + expect(Array.isArray(write?.Resource)).toBe(false); +}); + +test("a customer-named dedicated group is granted alongside the service namespace", () => { + const write = writeStatement( + executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [], { + cloudWatchConfig: { + logGroupName: "/company/agent-evaluations", + resultDestination: "DEDICATED_LOG_GROUP", + }, + }), + ); + + expect(write?.Resource).toEqual([ + SERVICE_RESULTS, + `arn:aws:logs:${REGION}:${ACCOUNT}:log-group:/company/agent-evaluations*`, + ]); + expect(write?.Action).toContain("logs:CreateLogGroup"); +}); + +test("SOURCE_LOG_GROUP grants writes to the groups the traces are read from", () => { + const write = writeStatement( + executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [], { + cloudWatchConfig: { resultDestination: "SOURCE_LOG_GROUP" }, + }), + ); + + expect(write?.Resource).toEqual([ + SERVICE_RESULTS, + `arn:aws:logs:${REGION}:${ACCOUNT}:log-group:/aws/bedrock-agentcore/runtimes/orders-agent-abc123*`, + ]); +}); + +test("a destination already inside the service namespace adds nothing", () => { + const write = writeStatement( + executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [], { + cloudWatchConfig: { + logGroupName: "/aws/bedrock-agentcore/evaluations/online-evaluations/results/default", + resultDestination: "DEDICATED_LOG_GROUP", + }, + }), + ); + + expect(write?.Resource).toBe(SERVICE_RESULTS); +}); + +test("changing the destination changes the policy name, so a re-scope is a new grant", () => { + const before = executionPolicy(REGION, ACCOUNT, LOG_GROUPS, []); + const after = executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [], { + cloudWatchConfig: { logGroupName: "/company/agent-evaluations" }, + }); + + expect(scopePolicyName(after)).not.toBe(scopePolicyName(before)); +}); diff --git a/src/core/onlineEvalExecutionRole.tsx b/src/core/onlineEvalExecutionRole.tsx index e02a6f40a..4c1896b6e 100644 --- a/src/core/onlineEvalExecutionRole.tsx +++ b/src/core/onlineEvalExecutionRole.tsx @@ -15,7 +15,6 @@ import { parseArn, resourceNameFromArn } from "./arn"; // evaluation results back to CloudWatch. When the caller doesn't bring one, // OnlineEvalClient provisions a per-config default here, scoped to the log // group(s) being sampled. Idempotent: an existing role is reused. -// // Each scope is stored as its own inline policy, named after a fingerprint of the // scope, so granting a new scope never overwrites the policy backing the current // one. IAM unions Allows across a role's inline policies, which lets an update @@ -83,10 +82,35 @@ function runtimeLogGroupPrefix(logGroupName: string): string { return match?.[1] ?? logGroupName; } +const SERVICE_RESULT_PREFIX = "/aws/bedrock-agentcore/evaluations/"; + +function resultWriteArns( + logs: string, + sampledArns: string[], + outputConfig: OnlineEvalResultDestination | undefined, +): string | string[] { + const arns = [`${logs}:${SERVICE_RESULT_PREFIX}*`]; + const cloudWatch = outputConfig?.cloudWatchConfig; + + if (cloudWatch?.resultDestination === "SOURCE_LOG_GROUP") { + arns.push(...sampledArns); + } else if ( + cloudWatch?.logGroupName && + !cloudWatch.logGroupName.startsWith(SERVICE_RESULT_PREFIX) + ) { + arns.push(`${logs}:${cloudWatch.logGroupName}*`); + } + + return arns.length === 1 ? arns[0]! : arns; +} + +export type OnlineEvalResultDestination = { + cloudWatchConfig?: { logGroupName?: string; resultDestination?: string } | undefined; +}; + // executionPolicy grants the permissions CreateOnlineEvaluationConfig validates // at creation time. Exported for assertion: the policy body is not observable // through the recorded IAM fixtures, whose responses are empty. -// // at creation time: Logs Insights query access over the sampled log groups plus // the `aws/spans` group that carries the actual trace spans, Bedrock model // invocation for LLM-as-a-Judge evaluators, Lambda invocation for code-based @@ -98,6 +122,7 @@ export function executionPolicy( accountId: string, logGroupNames: string[], kmsKeyArns: string[], + outputConfig?: OnlineEvalResultDestination, ): string { const logs = `arn:aws:logs:${region}:${accountId}:log-group`; const spansArn = `${logs}:aws/spans`; @@ -134,6 +159,8 @@ export function executionPolicy( Resource: [`${spansArn}*`, ...sampledArns], }, { + // logs:CreateLogGroup is needed because the service creates a + // customer-named result group that does not exist yet. Sid: "WriteEvaluationResults", Effect: "Allow", Action: [ @@ -142,7 +169,7 @@ export function executionPolicy( "logs:DescribeLogStreams", "logs:PutLogEvents", ], - Resource: `${logs}:/aws/bedrock-agentcore/evaluations/*`, + Resource: resultWriteArns(logs, sampledArns, outputConfig), }, { Sid: "IndexSpans", @@ -201,6 +228,11 @@ export function scopePolicyName(policyDocument: string): string { return `${POLICY_PREFIX}-${fingerprint(policyDocument)}`; } +export type GrantScopeOptions = { + roleName?: string; + outputConfig?: OnlineEvalResultDestination; +}; + // grantOnlineEvalScope creates the execution role for `configName` if it does not // exist and attaches the inline policy for this scope, returning the role ARN and // the policy name written. The caller revokes the superseded scope once whatever @@ -211,7 +243,7 @@ export async function grantOnlineEvalScope( region: string, logGroupNames: string[], kmsKeyArns: string[] = [], - roleName = onlineEvalExecutionRoleName(configName), + { roleName = onlineEvalExecutionRoleName(configName), outputConfig }: GrantScopeOptions = {}, ): Promise<{ roleArn: string; policyName: string }> { let roleArn: string; try { @@ -234,6 +266,7 @@ export async function grantOnlineEvalScope( accountIdFromRoleArn(roleArn), logGroupNames, kmsKeyArns, + outputConfig, ); const policyName = scopePolicyName(policyDocument); await iam.send( diff --git a/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.5e70e8cf174a7bc6.json b/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.5e70e8cf174a7bc6.json index 9a38e2ba6..9ea2f2ab0 100644 --- a/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.5e70e8cf174a7bc6.json +++ b/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.5e70e8cf174a7bc6.json @@ -1,14 +1,14 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_role_warn-2KNTGuGVDl", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_role_warn-2KNTGuGVDl", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_role_warn-tSVCZiBf4l", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_role_warn-tSVCZiBf4l", "createdAt": { - "$date": "2026-08-03T21:29:57.112Z" + "$date": "2026-09-15T15:11:36.526Z" }, "status": "CREATING", "executionStatus": "DISABLED", "outputConfig": { "cloudWatchConfig": { - "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_role_warn-2KNTGuGVDl" + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_role_warn-tSVCZiBf4l" } } } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.6b79e5be866896d6.json b/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.6b79e5be866896d6.json index 53629d6dd..724eb49bc 100644 --- a/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.6b79e5be866896d6.json +++ b/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.6b79e5be866896d6.json @@ -1,14 +1,14 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_kms-bG4DUW3Ua5", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_kms-bG4DUW3Ua5", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_kms-15ciiv2UoV", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_kms-15ciiv2UoV", "createdAt": { - "$date": "2026-08-03T21:29:51.133Z" + "$date": "2026-09-15T15:11:30.185Z" }, "status": "CREATING", "executionStatus": "DISABLED", "outputConfig": { "cloudWatchConfig": { - "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_kms-bG4DUW3Ua5" + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_kms-15ciiv2UoV" } } } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.bdca68c635390f67.json b/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.f9d2b8ebfad2219b.json similarity index 57% rename from src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.bdca68c635390f67.json rename to src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.f9d2b8ebfad2219b.json index c4f767ed4..eab1c3308 100644 --- a/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.bdca68c635390f67.json +++ b/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.f9d2b8ebfad2219b.json @@ -1,14 +1,12 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-5gZw987afd", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-5gZw987afd", "createdAt": { - "$date": "2026-08-03T21:29:43.069Z" + "$date": "2026-09-15T15:11:21.000Z" }, "status": "CREATING", "executionStatus": "DISABLED", "outputConfig": { - "cloudWatchConfig": { - "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_fixture-vYkaD93sFk" - } + "cloudWatchConfig": {} } } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.fa01de9ac60dc685.json b/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.2ae2ccb0272b469e.json similarity index 83% rename from src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.fa01de9ac60dc685.json rename to src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.2ae2ccb0272b469e.json index 2feaa4398..d3cde1ddf 100644 --- a/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.fa01de9ac60dc685.json +++ b/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.2ae2ccb0272b469e.json @@ -1,5 +1,5 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_role_warn-2KNTGuGVDl", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_role_warn-2KNTGuGVDl", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_role_warn-tSVCZiBf4l", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_role_warn-tSVCZiBf4l", "status": "DELETING" } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.77430112d0720f6.json b/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.d1856366f3636123.json similarity index 61% rename from src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.77430112d0720f6.json rename to src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.d1856366f3636123.json index 420108a61..c11c730d1 100644 --- a/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.77430112d0720f6.json +++ b/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.d1856366f3636123.json @@ -1,5 +1,5 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_kms-bG4DUW3Ua5", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_kms-bG4DUW3Ua5", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_kms-15ciiv2UoV", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_kms-15ciiv2UoV", "status": "DELETING" } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.a952ff6787c06557.json b/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.e3250f36c85f44ba.json similarity index 84% rename from src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.a952ff6787c06557.json rename to src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.e3250f36c85f44ba.json index da16e6717..3468e930f 100644 --- a/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.a952ff6787c06557.json +++ b/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.e3250f36c85f44ba.json @@ -1,5 +1,5 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-5gZw987afd", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-5gZw987afd", "status": "DELETING" } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/GetAgentRuntimeCommand.80bb567edcaa4bb1.json b/src/handlers/eval/online-eval/__fixtures__/GetAgentRuntimeCommand.80bb567edcaa4bb1.json index 5e8b39035..a8ad68114 100644 --- a/src/handlers/eval/online-eval/__fixtures__/GetAgentRuntimeCommand.80bb567edcaa4bb1.json +++ b/src/handlers/eval/online-eval/__fixtures__/GetAgentRuntimeCommand.80bb567edcaa4bb1.json @@ -10,14 +10,14 @@ "$date": "2026-03-24T16:13:08.198Z" }, "roleArn": "arn:aws:iam::725476964917:role/AgentCore-myimport-defaul-ApplicationAgentTestAgent-tJFjyd6jLIOn", - "networkConfiguration": { - "networkMode": "PUBLIC" - }, "status": "READY", "lifecycleConfiguration": { "idleRuntimeSessionTimeout": 900, "maxLifetime": 28800 }, + "networkConfiguration": { + "networkMode": "PUBLIC" + }, "description": "AgentCore Runtime: myimport_testAgent_Agent", "workloadIdentityDetails": { "workloadIdentityArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:workload-identity-directory/default/workload-identity/testAgent_Agent-wm9hYBD93Y" diff --git a/src/handlers/eval/online-eval/__fixtures__/GetAgentRuntimeCommand.d729967662bebc9f.json b/src/handlers/eval/online-eval/__fixtures__/GetAgentRuntimeCommand.d729967662bebc9f.json index 46ce84be3..05862e44b 100644 --- a/src/handlers/eval/online-eval/__fixtures__/GetAgentRuntimeCommand.d729967662bebc9f.json +++ b/src/handlers/eval/online-eval/__fixtures__/GetAgentRuntimeCommand.d729967662bebc9f.json @@ -10,14 +10,14 @@ "$date": "2026-06-17T22:09:21.093Z" }, "roleArn": "arn:aws:iam::725476964917:role/AgentCore-ABVfyLatest-def-ApplicationAgentABVfyLate-b6b570G88FJ3", - "networkConfiguration": { - "networkMode": "PUBLIC" - }, "status": "READY", "lifecycleConfiguration": { "idleRuntimeSessionTimeout": 900, "maxLifetime": 28800 }, + "networkConfiguration": { + "networkMode": "PUBLIC" + }, "description": "AgentCore Runtime: ABVfyLatest_ABVfyLatest", "workloadIdentityDetails": { "workloadIdentityArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:workload-identity-directory/default/workload-identity/ABVfyLatest_ABVfyLatest-PFLr353QVA" diff --git a/src/handlers/eval/online-eval/__fixtures__/GetEvaluatorCommand.5ee939ef1fda1ca7.json b/src/handlers/eval/online-eval/__fixtures__/GetEvaluatorCommand.5ee939ef1fda1ca7.json index 47000de38..674771ee2 100644 --- a/src/handlers/eval/online-eval/__fixtures__/GetEvaluatorCommand.5ee939ef1fda1ca7.json +++ b/src/handlers/eval/online-eval/__fixtures__/GetEvaluatorCommand.5ee939ef1fda1ca7.json @@ -2,6 +2,26 @@ "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness", "evaluatorId": "Builtin.Correctness", "evaluatorName": "Builtin.Correctness", + "evaluatorConfig": { + "llmAsAJudge": { + "ratingScale": { + "numerical": [ + { + "value": 0, + "label": "Incorrect" + }, + { + "value": 0.5, + "label": "Partially correct" + }, + { + "value": 1, + "label": "Correct" + } + ] + } + } + }, "level": "TRACE", "status": "ACTIVE", "createdAt": { @@ -10,5 +30,8 @@ "updatedAt": { "$date": "2024-10-22T00:00:00.000Z" }, - "kmsKeyArn": "arn:aws:kms:us-west-2:725476964917:key/31a2dd2f-c8a0-42b8-9f52-12e5ecb22468" -} + "description": "Response Quality Metric. Evaluates whether the information in the agent's response is factually accurate", + "evaluatorType": "Builtin", + "provider": "AWS", + "lockedForModification": true +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json b/src/handlers/eval/online-eval/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json index 1825df228..575c78948 100644 --- a/src/handlers/eval/online-eval/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json +++ b/src/handlers/eval/online-eval/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json @@ -47,5 +47,7 @@ "$date": "2024-10-22T00:00:00.000Z" }, "description": "Response Quality Metric. Evaluates from user's perspective how useful and valuable the agent's response is", + "evaluatorType": "Builtin", + "provider": "AWS", "lockedForModification": true } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.fa01de9ac60dc685.json b/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.2ae2ccb0272b469e.json similarity index 72% rename from src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.fa01de9ac60dc685.json rename to src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.2ae2ccb0272b469e.json index 791fac7d0..85afdec7a 100644 --- a/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.fa01de9ac60dc685.json +++ b/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.2ae2ccb0272b469e.json @@ -1,6 +1,6 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_role_warn-2KNTGuGVDl", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_role_warn-2KNTGuGVDl", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_role_warn-tSVCZiBf4l", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_role_warn-tSVCZiBf4l", "onlineEvaluationConfigName": "agentcore_cli_online_eval_role_warn", "rule": { "samplingConfig": { @@ -9,21 +9,21 @@ }, "dataSourceConfig": { "cloudWatchLogs": { - "logGroupNames": [ - "/aws/bedrock-agentcore/runtimes/ABVfyLatest_ABVfyLatest-PFLr353QVA-DEFAULT" - ], "serviceNames": [ - "ABVfyLatest_ABVfyLatest.DEFAULT" + "testAgent_Agent.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/runtimes/testAgent_Agent-wm9hYBD93Y-DEFAULT" ] } }, "status": "ACTIVE", "executionStatus": "DISABLED", "createdAt": { - "$date": "2026-08-03T21:29:57.112Z" + "$date": "2026-09-15T15:11:36.526Z" }, "updatedAt": { - "$date": "2026-08-03T21:30:02.741Z" + "$date": "2026-09-15T15:11:36.665Z" }, "evaluators": [ { @@ -32,7 +32,8 @@ ], "outputConfig": { "cloudWatchConfig": { - "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_role_warn-2KNTGuGVDl" + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_role_warn-tSVCZiBf4l", + "resultDestination": "DEDICATED_LOG_GROUP" } }, "evaluationExecutionRoleArn": "arn:aws:iam::725476964917:role/AgentCoreEvalsSDK-us-west-2-a6864eb339" diff --git a/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.77430112d0720f6.json b/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.d1856366f3636123.json similarity index 78% rename from src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.77430112d0720f6.json rename to src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.d1856366f3636123.json index cbb739904..97ab5d9fc 100644 --- a/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.77430112d0720f6.json +++ b/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.d1856366f3636123.json @@ -1,6 +1,6 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_kms-bG4DUW3Ua5", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_kms-bG4DUW3Ua5", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_kms-15ciiv2UoV", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_kms-15ciiv2UoV", "onlineEvaluationConfigName": "agentcore_cli_online_eval_kms", "rule": { "samplingConfig": { @@ -9,21 +9,21 @@ }, "dataSourceConfig": { "cloudWatchLogs": { - "logGroupNames": [ - "/aws/bedrock-agentcore/runtimes/testAgent_Agent-wm9hYBD93Y-DEFAULT" - ], "serviceNames": [ "testAgent_Agent.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/runtimes/testAgent_Agent-wm9hYBD93Y-DEFAULT" ] } }, "status": "ACTIVE", "executionStatus": "DISABLED", "createdAt": { - "$date": "2026-08-03T21:29:51.133Z" + "$date": "2026-09-15T15:11:30.185Z" }, "updatedAt": { - "$date": "2026-08-03T21:29:51.288Z" + "$date": "2026-09-15T15:11:30.386Z" }, "evaluators": [ { @@ -35,7 +35,8 @@ ], "outputConfig": { "cloudWatchConfig": { - "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_kms-bG4DUW3Ua5" + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_kms-15ciiv2UoV", + "resultDestination": "DEDICATED_LOG_GROUP" } }, "evaluationExecutionRoleArn": "arn:aws:iam::725476964917:role/AgentCoreOnlineEval-agentcore_cli_online_eval_kms" diff --git a/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.a952ff6787c06557.json b/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.e3250f36c85f44ba.json similarity index 81% rename from src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.a952ff6787c06557.json rename to src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.e3250f36c85f44ba.json index 5355a73df..7d7d6fd3f 100644 --- a/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.a952ff6787c06557.json +++ b/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.e3250f36c85f44ba.json @@ -1,6 +1,6 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-5gZw987afd", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-5gZw987afd", "onlineEvaluationConfigName": "agentcore_cli_online_eval_fixture", "rule": { "samplingConfig": { @@ -12,22 +12,23 @@ }, "dataSourceConfig": { "cloudWatchLogs": { - "logGroupNames": [ - "/aws/bedrock-agentcore/runtimes/testAgent_Agent-wm9hYBD93Y-DEFAULT" - ], "serviceNames": [ "testAgent_Agent.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/runtimes/testAgent_Agent-wm9hYBD93Y-DEFAULT" ] } }, "status": "ACTIVE", "executionStatus": "DISABLED", "createdAt": { - "$date": "2026-08-03T21:29:43.069Z" + "$date": "2026-09-15T15:11:21.000Z" }, "updatedAt": { - "$date": "2026-08-03T21:29:44.358Z" + "$date": "2026-09-15T15:11:22.528Z" }, + "description": "quality monitor for prod traffic", "evaluators": [ { "evaluatorId": "Builtin.Helpfulness" @@ -35,7 +36,7 @@ ], "outputConfig": { "cloudWatchConfig": { - "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_fixture-vYkaD93sFk" + "resultDestination": "SOURCE_LOG_GROUP" } }, "evaluationExecutionRoleArn": "arn:aws:iam::725476964917:role/AgentCoreOnlineEval-agentcore_cli_online_eval_fixture" diff --git a/src/handlers/eval/online-eval/__fixtures__/GetRoleCommand.1ba024c017a5c69d.json b/src/handlers/eval/online-eval/__fixtures__/GetRoleCommand.1ba024c017a5c69d.json index 6f9666e79..14874e586 100644 --- a/src/handlers/eval/online-eval/__fixtures__/GetRoleCommand.1ba024c017a5c69d.json +++ b/src/handlers/eval/online-eval/__fixtures__/GetRoleCommand.1ba024c017a5c69d.json @@ -12,7 +12,7 @@ "MaxSessionDuration": 3600, "RoleLastUsed": { "LastUsedDate": { - "$date": "2026-08-03T19:57:02.000Z" + "$date": "2026-08-03T21:29:51.000Z" }, "Region": "us-west-2" } diff --git a/src/handlers/eval/online-eval/__fixtures__/GetRoleCommand.f532eb8675abbed5.json b/src/handlers/eval/online-eval/__fixtures__/GetRoleCommand.f532eb8675abbed5.json index a9222a5c0..0c82dc92c 100644 --- a/src/handlers/eval/online-eval/__fixtures__/GetRoleCommand.f532eb8675abbed5.json +++ b/src/handlers/eval/online-eval/__fixtures__/GetRoleCommand.f532eb8675abbed5.json @@ -12,7 +12,7 @@ "MaxSessionDuration": 3600, "RoleLastUsed": { "LastUsedDate": { - "$date": "2026-08-03T19:56:55.000Z" + "$date": "2026-08-03T21:29:44.000Z" }, "Region": "us-west-2" } diff --git a/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.23f97c9dcdd6350b.json b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.23f97c9dcdd6350b.json index d56c1eb32..d1f83c4a1 100644 --- a/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.23f97c9dcdd6350b.json +++ b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.23f97c9dcdd6350b.json @@ -274,16 +274,16 @@ } }, { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-5gZw987afd", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-5gZw987afd", "onlineEvaluationConfigName": "agentcore_cli_online_eval_fixture", - "status": "CREATING", + "status": "ACTIVE", "executionStatus": "DISABLED", "createdAt": { - "$date": "2026-08-03T21:29:43.069Z" + "$date": "2026-09-15T15:11:21.000Z" }, "updatedAt": { - "$date": "2026-08-03T21:29:43.069Z" + "$date": "2026-09-15T15:11:21.210Z" } }, { @@ -412,6 +412,24 @@ "$date": "2026-04-21T22:47:52.237Z" } }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/oiVerify1_prodInsights-rPYFuF2WsE", + "onlineEvaluationConfigId": "oiVerify1_prodInsights-rPYFuF2WsE", + "onlineEvaluationConfigName": "oiVerify1_prodInsights", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-08-21T16:49:04.244Z" + }, + "updatedAt": { + "$date": "2026-08-21T16:49:04.485Z" + }, + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ] + }, { "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/temp-5Dz9Ox6yMC", "onlineEvaluationConfigId": "temp-5Dz9Ox6yMC", diff --git a/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.e001e6ee1696fc1.json b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.7085725f3456218e.json similarity index 58% rename from src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.e001e6ee1696fc1.json rename to src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.7085725f3456218e.json index 2fae4108e..4a2927933 100644 --- a/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.e001e6ee1696fc1.json +++ b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.7085725f3456218e.json @@ -14,5 +14,5 @@ } } ], - "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAEv6LZkoBF2ff9apGPQDI1QAAABLTCCASkGCSqGSIb3DQEHBqCCARowggEWAgEAMIIBDwYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAzndfN1G5lOwU0OoKcCARCAgeFptcFMzCJHdqqbuwga6ImIwdP9YJyQW82pmhDStQefIKBE5DNOnEre+0wrAPnlIrMkImcj2K+ToMSe6Sy+j3NkrDY//zdwlE9izmkzoP9qFzD6nmtoLSVqOBKb/1o6I2019xZmJETD4SMO8FBO8Xw/jPKFyeopIrn5cAQ8/HcV2VhQ1InbeRBErVARDbXLWy/zzFXNQXM5+yXHmpJNKhafJ402KhznsNFfMN3HI+HcTNw64EiqX8ChbSoGW0JBpXaUJ7ZKHr+pocmVVdJpbNgxw4rHJK3WWENG5YTdWAXbNZs=" + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAHVPDOZZgGDjPioe28DDKn9AAABLTCCASkGCSqGSIb3DQEHBqCCARowggEWAgEAMIIBDwYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAxbfqHBsJHhexxeAzcCARCAgeGrw32yAYu5qYvC/UxEPSulAopvMhPHrrECfWLCyDbK8xevzkZzXLNqPjvyfMI1f4vqFhT/f5m9TxdfM85DZXF6MHfhuQmczyVrkJ6nbVT0gogaEFpgfQyYrjPq/e2KxAfxxJBid6LRWnuiYMwGhvHSlFH2DuVkuOJ5r2DHF408hTGJmyTl7xFCOH9DiopXFecnatxj67eWc4dEQCmfNYKjtKEjMmwOZP0EorRlma0kExIo1z8m26qjlPE5/AJHh8UCIIcGmP/RIH5/HVmwllGOhoB57DmpHL0+Ssh6oaubawo=" } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.7d2e22c637f6b633.json b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.7d2e22c637f6b633.json index 7001a2c94..3eb2f4e89 100644 --- a/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.7d2e22c637f6b633.json +++ b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.7d2e22c637f6b633.json @@ -14,5 +14,5 @@ } } ], - "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAHs3Tc/JEuHsHfRfV7OwMsqAAABKjCCASYGCSqGSIb3DQEHBqCCARcwggETAgEAMIIBDAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAxvGMxcYVwGUaMn490CARCAgd6Grki8j/DTGYrCjoz0pPponIAeCwB4DOt6kzm9O+CUSYZRSXfx2z9ik6NeWb7yunR708ju0A0Ub4UZP+cJzOIh1MfdU79KTB5BOad/OwSGPMEMLQ9yKvXLUhXwA4p8fiRkO6fk49N+foUK54IBCF+B/mLuG2S1a/8rNByu0qD5xA/oRv2GNJdOnf+8bS8jNEiyQyEhiQR7e6dNO8karHz6HQx3ziBj7AG6peN8YxWGa1b60/P76eg0GP1IYZ0Yq2jGXyzmKG4AXAxkzq7opAOm8uASql2sxR++fXDrleM=" + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAG2LjxPJ+EJVOED31V+ZHVXAAABKjCCASYGCSqGSIb3DQEHBqCCARcwggETAgEAMIIBDAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAyPzCs7MwByuqxtRuoCARCAgd4dsac1ZBLMyEBDeLAY77mWSA8WQrm88f0RUhTGSSWQcbk0LyRzlhZBBWXsQ9QFM5rWTpre3JDrKFHXxkxlH3DcIIcnv7eByMix9Fyt2VadgDkHjA3d2A1mA4ltVL4ludGUQe5oN2TwYKPUKzqAB6o31PO+Dfyy6MJuWbr0l4OiIWR0ZTvDFWBhcDbCD8MwppNPI9C3B/7sQxUC0FuDUX92SR2yKs+VSNLBs6tU8tievQ7+Yvv+kOXwa1IifC2lUPBlpa3hnIP2XcAnM311MZnQUGmKhH71+n7YFCbaATU=" } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.5a7fd2344c4d345a.json b/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.5a7fd2344c4d345a.json deleted file mode 100644 index 0967ef424..000000000 --- a/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.5a7fd2344c4d345a.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.a783917ba38c39cf.json b/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.a783917ba38c39cf.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.a783917ba38c39cf.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.ace7666da95be80c.json b/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.ace7666da95be80c.json deleted file mode 100644 index 0967ef424..000000000 --- a/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.ace7666da95be80c.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.cb047843e517ce33.json b/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.cb047843e517ce33.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.cb047843e517ce33.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.2cc6a186c2d4961e.json b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.6aabff07c1e09212.json similarity index 77% rename from src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.2cc6a186c2d4961e.json rename to src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.6aabff07c1e09212.json index 7a0e96308..a17bb3e90 100644 --- a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.2cc6a186c2d4961e.json +++ b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.6aabff07c1e09212.json @@ -1,8 +1,8 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-5gZw987afd", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-5gZw987afd", "updatedAt": { - "$date": "2026-08-03T21:29:49.831Z" + "$date": "2026-09-15T15:11:28.343Z" }, "status": "UPDATING", "executionStatus": "DISABLED" diff --git a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.1c674dc74c2a961c.json b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.893e75f9aa31c2dd.json similarity index 76% rename from src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.1c674dc74c2a961c.json rename to src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.893e75f9aa31c2dd.json index ef9b0b88d..5e4328707 100644 --- a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.1c674dc74c2a961c.json +++ b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.893e75f9aa31c2dd.json @@ -1,8 +1,8 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_role_warn-2KNTGuGVDl", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_role_warn-2KNTGuGVDl", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_role_warn-tSVCZiBf4l", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_role_warn-tSVCZiBf4l", "updatedAt": { - "$date": "2026-08-03T21:30:08.342Z" + "$date": "2026-09-15T15:11:42.254Z" }, "status": "ACTIVE", "executionStatus": "DISABLED" diff --git a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.4ba036918b84f090.json b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.8b9d669be4e53776.json similarity index 77% rename from src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.4ba036918b84f090.json rename to src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.8b9d669be4e53776.json index bf173abca..6332cd144 100644 --- a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.4ba036918b84f090.json +++ b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.8b9d669be4e53776.json @@ -1,8 +1,8 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-5gZw987afd", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-5gZw987afd", "updatedAt": { - "$date": "2026-08-03T21:29:44.358Z" + "$date": "2026-09-15T15:11:22.528Z" }, "status": "ACTIVE", "executionStatus": "DISABLED" diff --git a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.29bcd2abed379bc9.json b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.a26788dd06a0fe0d.json similarity index 77% rename from src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.29bcd2abed379bc9.json rename to src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.a26788dd06a0fe0d.json index 2743b289c..a063ca0b7 100644 --- a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.29bcd2abed379bc9.json +++ b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.a26788dd06a0fe0d.json @@ -1,8 +1,8 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-5gZw987afd", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-5gZw987afd", "updatedAt": { - "$date": "2026-08-03T21:29:44.625Z" + "$date": "2026-09-15T15:11:23.030Z" }, "status": "UPDATING", "executionStatus": "ENABLED" diff --git a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.d88366abe9f2729b.json b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.d88366abe9f2729b.json deleted file mode 100644 index 08c6ab0c2..000000000 --- a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.d88366abe9f2729b.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_role_warn-2KNTGuGVDl", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_role_warn-2KNTGuGVDl", - "updatedAt": { - "$date": "2026-08-03T21:30:02.741Z" - }, - "status": "ACTIVE", - "executionStatus": "DISABLED" -} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/create.golden.json b/src/handlers/eval/online-eval/__fixtures__/create.golden.json index edfd4d2e5..c242d2484 100644 --- a/src/handlers/eval/online-eval/__fixtures__/create.golden.json +++ b/src/handlers/eval/online-eval/__fixtures__/create.golden.json @@ -1,12 +1,10 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", - "createdAt": "2026-08-03T21:29:43.069Z", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-5gZw987afd", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-5gZw987afd", + "createdAt": "2026-09-15T15:11:21.000Z", "status": "CREATING", "executionStatus": "DISABLED", "outputConfig": { - "cloudWatchConfig": { - "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_fixture-vYkaD93sFk" - } + "cloudWatchConfig": {} } } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/delete.golden.json b/src/handlers/eval/online-eval/__fixtures__/delete.golden.json index da16e6717..3468e930f 100644 --- a/src/handlers/eval/online-eval/__fixtures__/delete.golden.json +++ b/src/handlers/eval/online-eval/__fixtures__/delete.golden.json @@ -1,5 +1,5 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-5gZw987afd", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-5gZw987afd", "status": "DELETING" } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/get.golden.json b/src/handlers/eval/online-eval/__fixtures__/get.golden.json index 574ee3501..c95911733 100644 --- a/src/handlers/eval/online-eval/__fixtures__/get.golden.json +++ b/src/handlers/eval/online-eval/__fixtures__/get.golden.json @@ -1,6 +1,6 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-5gZw987afd", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-5gZw987afd", "onlineEvaluationConfigName": "agentcore_cli_online_eval_fixture", "rule": { "samplingConfig": { @@ -12,18 +12,19 @@ }, "dataSourceConfig": { "cloudWatchLogs": { - "logGroupNames": [ - "/aws/bedrock-agentcore/runtimes/testAgent_Agent-wm9hYBD93Y-DEFAULT" - ], "serviceNames": [ "testAgent_Agent.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/runtimes/testAgent_Agent-wm9hYBD93Y-DEFAULT" ] } }, "status": "ACTIVE", "executionStatus": "DISABLED", - "createdAt": "2026-08-03T21:29:43.069Z", - "updatedAt": "2026-08-03T21:29:44.358Z", + "createdAt": "2026-09-15T15:11:21.000Z", + "updatedAt": "2026-09-15T15:11:22.528Z", + "description": "quality monitor for prod traffic", "evaluators": [ { "evaluatorId": "Builtin.Helpfulness" @@ -31,7 +32,7 @@ ], "outputConfig": { "cloudWatchConfig": { - "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_fixture-vYkaD93sFk" + "resultDestination": "SOURCE_LOG_GROUP" } }, "evaluationExecutionRoleArn": "arn:aws:iam::725476964917:role/AgentCoreOnlineEval-agentcore_cli_online_eval_fixture" diff --git a/src/handlers/eval/online-eval/__fixtures__/list-page-1.golden.json b/src/handlers/eval/online-eval/__fixtures__/list-page-1.golden.json index 50e4154d1..64f70b30d 100644 --- a/src/handlers/eval/online-eval/__fixtures__/list-page-1.golden.json +++ b/src/handlers/eval/online-eval/__fixtures__/list-page-1.golden.json @@ -10,5 +10,5 @@ "updatedAt": "2026-06-17T22:10:11.792Z" } ], - "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAHs3Tc/JEuHsHfRfV7OwMsqAAABKjCCASYGCSqGSIb3DQEHBqCCARcwggETAgEAMIIBDAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAxvGMxcYVwGUaMn490CARCAgd6Grki8j/DTGYrCjoz0pPponIAeCwB4DOt6kzm9O+CUSYZRSXfx2z9ik6NeWb7yunR708ju0A0Ub4UZP+cJzOIh1MfdU79KTB5BOad/OwSGPMEMLQ9yKvXLUhXwA4p8fiRkO6fk49N+foUK54IBCF+B/mLuG2S1a/8rNByu0qD5xA/oRv2GNJdOnf+8bS8jNEiyQyEhiQR7e6dNO8karHz6HQx3ziBj7AG6peN8YxWGa1b60/P76eg0GP1IYZ0Yq2jGXyzmKG4AXAxkzq7opAOm8uASql2sxR++fXDrleM=" + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAG2LjxPJ+EJVOED31V+ZHVXAAABKjCCASYGCSqGSIb3DQEHBqCCARcwggETAgEAMIIBDAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAyPzCs7MwByuqxtRuoCARCAgd4dsac1ZBLMyEBDeLAY77mWSA8WQrm88f0RUhTGSSWQcbk0LyRzlhZBBWXsQ9QFM5rWTpre3JDrKFHXxkxlH3DcIIcnv7eByMix9Fyt2VadgDkHjA3d2A1mA4ltVL4ludGUQe5oN2TwYKPUKzqAB6o31PO+Dfyy6MJuWbr0l4OiIWR0ZTvDFWBhcDbCD8MwppNPI9C3B/7sQxUC0FuDUX92SR2yKs+VSNLBs6tU8tievQ7+Yvv+kOXwa1IifC2lUPBlpa3hnIP2XcAnM311MZnQUGmKhH71+n7YFCbaATU=" } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/list-page-2.golden.json b/src/handlers/eval/online-eval/__fixtures__/list-page-2.golden.json index 9f430050c..d6e1e585e 100644 --- a/src/handlers/eval/online-eval/__fixtures__/list-page-2.golden.json +++ b/src/handlers/eval/online-eval/__fixtures__/list-page-2.golden.json @@ -10,5 +10,5 @@ "updatedAt": "2026-06-17T22:10:12.095Z" } ], - "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAEv6LZkoBF2ff9apGPQDI1QAAABLTCCASkGCSqGSIb3DQEHBqCCARowggEWAgEAMIIBDwYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAzndfN1G5lOwU0OoKcCARCAgeFptcFMzCJHdqqbuwga6ImIwdP9YJyQW82pmhDStQefIKBE5DNOnEre+0wrAPnlIrMkImcj2K+ToMSe6Sy+j3NkrDY//zdwlE9izmkzoP9qFzD6nmtoLSVqOBKb/1o6I2019xZmJETD4SMO8FBO8Xw/jPKFyeopIrn5cAQ8/HcV2VhQ1InbeRBErVARDbXLWy/zzFXNQXM5+yXHmpJNKhafJ402KhznsNFfMN3HI+HcTNw64EiqX8ChbSoGW0JBpXaUJ7ZKHr+pocmVVdJpbNgxw4rHJK3WWENG5YTdWAXbNZs=" + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAHVPDOZZgGDjPioe28DDKn9AAABLTCCASkGCSqGSIb3DQEHBqCCARowggEWAgEAMIIBDwYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAxbfqHBsJHhexxeAzcCARCAgeGrw32yAYu5qYvC/UxEPSulAopvMhPHrrECfWLCyDbK8xevzkZzXLNqPjvyfMI1f4vqFhT/f5m9TxdfM85DZXF6MHfhuQmczyVrkJ6nbVT0gogaEFpgfQyYrjPq/e2KxAfxxJBid6LRWnuiYMwGhvHSlFH2DuVkuOJ5r2DHF408hTGJmyTl7xFCOH9DiopXFecnatxj67eWc4dEQCmfNYKjtKEjMmwOZP0EorRlma0kExIo1z8m26qjlPE5/AJHh8UCIIcGmP/RIH5/HVmwllGOhoB57DmpHL0+Ssh6oaubawo=" } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/list.golden.json b/src/handlers/eval/online-eval/__fixtures__/list.golden.json index e94eab176..575bca1a6 100644 --- a/src/handlers/eval/online-eval/__fixtures__/list.golden.json +++ b/src/handlers/eval/online-eval/__fixtures__/list.golden.json @@ -190,13 +190,13 @@ "updatedAt": "2026-07-06T18:21:28.312Z" }, { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-5gZw987afd", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-5gZw987afd", "onlineEvaluationConfigName": "agentcore_cli_online_eval_fixture", - "status": "CREATING", + "status": "ACTIVE", "executionStatus": "DISABLED", - "createdAt": "2026-08-03T21:29:43.069Z", - "updatedAt": "2026-08-03T21:29:43.069Z" + "createdAt": "2026-09-15T15:11:21.000Z", + "updatedAt": "2026-09-15T15:11:21.210Z" }, { "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/bugbashagent_abtesteval-53zYJX8x4X", @@ -292,6 +292,20 @@ "createdAt": "2026-04-21T22:47:43.896Z", "updatedAt": "2026-04-21T22:47:52.237Z" }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/oiVerify1_prodInsights-rPYFuF2WsE", + "onlineEvaluationConfigId": "oiVerify1_prodInsights-rPYFuF2WsE", + "onlineEvaluationConfigName": "oiVerify1_prodInsights", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-08-21T16:49:04.244Z", + "updatedAt": "2026-08-21T16:49:04.485Z", + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ] + }, { "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/temp-5Dz9Ox6yMC", "onlineEvaluationConfigId": "temp-5Dz9Ox6yMC", diff --git a/src/handlers/eval/online-eval/__fixtures__/pause.golden.json b/src/handlers/eval/online-eval/__fixtures__/pause.golden.json index 49df9ed01..e5349aa14 100644 --- a/src/handlers/eval/online-eval/__fixtures__/pause.golden.json +++ b/src/handlers/eval/online-eval/__fixtures__/pause.golden.json @@ -1,7 +1,7 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", - "updatedAt": "2026-08-03T21:29:49.831Z", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-5gZw987afd", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-5gZw987afd", + "updatedAt": "2026-09-15T15:11:28.343Z", "status": "UPDATING", "executionStatus": "DISABLED" } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/resume.golden.json b/src/handlers/eval/online-eval/__fixtures__/resume.golden.json index 7b4f0fc12..8da46e43d 100644 --- a/src/handlers/eval/online-eval/__fixtures__/resume.golden.json +++ b/src/handlers/eval/online-eval/__fixtures__/resume.golden.json @@ -1,7 +1,7 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", - "updatedAt": "2026-08-03T21:29:44.625Z", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-5gZw987afd", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-5gZw987afd", + "updatedAt": "2026-09-15T15:11:23.030Z", "status": "UPDATING", "executionStatus": "ENABLED" } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/update.golden.json b/src/handlers/eval/online-eval/__fixtures__/update.golden.json index 6fa397d72..17adb4b20 100644 --- a/src/handlers/eval/online-eval/__fixtures__/update.golden.json +++ b/src/handlers/eval/online-eval/__fixtures__/update.golden.json @@ -1,7 +1,7 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", - "updatedAt": "2026-08-03T21:29:44.358Z", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-5gZw987afd", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-5gZw987afd", + "updatedAt": "2026-09-15T15:11:22.528Z", "status": "ACTIVE", "executionStatus": "DISABLED" } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/create/index.tsx b/src/handlers/eval/online-eval/create/index.tsx index 273b95c7b..be28e04e1 100644 --- a/src/handlers/eval/online-eval/create/index.tsx +++ b/src/handlers/eval/online-eval/create/index.tsx @@ -5,9 +5,24 @@ import { InputValidationError } from "../../../../errors"; import { JsonRendererKey } from "../../../../tui"; import { SourceResolver, type AppIO } from "../../../../io"; import type { Core } from "../../../types"; -import { assertMutuallyExclusiveFlags, coreOptsFromCtx, parseJsonFlag } from "../../../utils"; +import { + assertMutuallyExclusiveFlags, + coreOptsFromCtx, + parseJsonFlag, + parseJsonFlagWithSchema, +} from "../../../utils"; import { filtersHelp } from "../filtersHelp"; import { onlineEvalDataSourceConfigHelp } from "../dataSourceConfigHelp"; +import { OnlineEvalOutputConfigFlag } from "../outputConfig"; +import { TagsSchema } from "../../../../projectSchemas/tags"; + +const tagsHelp = `(JSON: map of string to string) +Tags applied to the online evaluation configuration. + +Accepts inline JSON, file://, or - to read stdin. + +Example: + --tags '{"team":"ml-platform","env":"prod"}'`; const CONFIGURATION = "Configuration:"; const SESSION_SOURCE = "Session source (choose exactly one):"; @@ -35,6 +50,10 @@ export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) => z.enum(["true", "false"]).optional(), { group: CONFIGURATION }, ), + flag("tags", "resource tags (JSON object of key/value strings)", z.string().optional(), { + group: CONFIGURATION, + help: tagsHelp, + }), flag("agent", "harness ID or Runtime ID whose traffic to sample", z.string().optional(), { group: SESSION_SOURCE, }), @@ -69,6 +88,7 @@ export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) => group: EVALUATION, help: filtersHelp, }), + ...OnlineEvalOutputConfigFlag.flags, flag( "role-arn", "IAM role the online evaluation assumes (default auto-provisioned)", @@ -97,10 +117,21 @@ export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) => throw new InputValidationError("'--endpoint' can only be used with '--agent'"); } + // One resolver shared across every stdin-capable flag (--tags, --filters, + // --data-source-config, --output-config) so a second `-` is rejected + // rather than reading empty after the first drains stdin. const source = new SourceResolver({ stdin: io.stdin }); + const outputConfig = await OnlineEvalOutputConfigFlag.resolve(flags["output-config"], source); + const tags = parseJsonFlagWithSchema( + "tags", + await source.resolveText("tags", flags["tags"]), + TagsSchema, + ); const common = { name: flags["name"], description: flags["description"], + tags, + outputConfig, samplingRate: flags["sampling-rate"], sessionTimeoutMinutes: flags["session-timeout-minutes"], filters: parseJsonFlag( diff --git a/src/handlers/eval/online-eval/online-eval.test.tsx b/src/handlers/eval/online-eval/online-eval.test.tsx index d315b0695..55fb95748 100644 --- a/src/handlers/eval/online-eval/online-eval.test.tsx +++ b/src/handlers/eval/online-eval/online-eval.test.tsx @@ -123,6 +123,11 @@ describe("eval online-eval command hierarchy", () => { describe("online-eval CRUDL", () => { test("creates an online evaluation config from an agent", async () => { + // --output-config and --tags are carried by this recording so the request + // fixture (CreateOnlineEvaluationConfigCommand.*) is the assertion they reach + // the SDK unchanged, and the provisioned role's PutRolePolicy fixture proves + // the write scope widened to the destination. SOURCE_LOG_GROUP avoids creating + // a dedicated result group, keeping the recording residue-free. const stdout = await run([ "eval", "online-eval", @@ -139,6 +144,10 @@ describe("online-eval CRUDL", () => { "30", "--enable-on-create", "false", + "--output-config", + '{"cloudWatchConfig":{"resultDestination":"SOURCE_LOG_GROUP"}}', + "--tags", + '{"team":"agentcore-cli"}', ]); matchGolden(FIXTURES, "create.golden.json", stdout); @@ -179,6 +188,8 @@ describe("online-eval CRUDL", () => { // update merges over the current config because UpdateOnlineEvaluationConfig // replaces the whole `rule`; this asserts the unset fields survive the round trip. test("updates only the sampling rate, preserving the session timeout", async () => { + // --description and --output-config ride along so the update request fixture + // asserts they reach the SDK; update never touches the execution role. const stdout = await run([ "eval", "online-eval", @@ -187,6 +198,10 @@ describe("online-eval CRUDL", () => { configId, "--sampling-rate", "25", + "--description", + "quality monitor for prod traffic", + "--output-config", + '{"cloudWatchConfig":{"resultDestination":"SOURCE_LOG_GROUP"}}', ]); matchGolden(FIXTURES, "update.golden.json", stdout); @@ -326,6 +341,38 @@ describe("flag validation", () => { ).rejects.toThrow(/Invalid JSON for option '--data-source-config'/); }); + const CREATE_BASE = [ + "eval", + "online-eval", + "create", + "--name", + CONFIG_NAME, + "--agent", + FIXTURE_AGENT_ID, + "--evaluators", + FIXTURE_EVALUATOR_ID, + "--sampling-rate", + "10", + ]; + + test("create rejects malformed --output-config JSON", async () => { + await expect(run([...CREATE_BASE, "--output-config", "{not json"])).rejects.toThrow( + /Invalid JSON for option '--output-config'/, + ); + }); + + test("create rejects malformed --tags JSON", async () => { + await expect(run([...CREATE_BASE, "--tags", "{not json"])).rejects.toThrow( + /Invalid JSON for option '--tags'/, + ); + }); + + test("create rejects a non-string --tags value rather than passing it to the API", async () => { + await expect(run([...CREATE_BASE, "--tags", '{"team":42}'])).rejects.toThrow( + /Invalid value for option '--tags'/, + ); + }); + test("update rejects --endpoint together with --clear-endpoint", async () => { await expect( run([ diff --git a/src/handlers/eval/online-eval/outputConfig.tsx b/src/handlers/eval/online-eval/outputConfig.tsx new file mode 100644 index 000000000..b7da87905 --- /dev/null +++ b/src/handlers/eval/online-eval/outputConfig.tsx @@ -0,0 +1,59 @@ +import z from "zod"; +import { SourceResolver } from "../../../io"; +import { flag } from "../../../router"; +import { parseJsonFlag } from "../../utils"; +import type { OnlineEvalOutputConfig } from "../types"; + +const outputConfigHelp = `(JSON object) +Where evaluation results and metrics are written. Omit it and results go to the +service-managed default location. Only top-level key: cloudWatchConfig. + +Accepts inline JSON, file://, or - to read stdin. + +JSON syntax: + { + "cloudWatchConfig": { + "logGroupName": "string", // result log group; omit for + // SOURCE_LOG_GROUP, and it cannot sit + // under /aws/bedrock-agentcore/evaluations/ + "metricsNamespace": "string", // CloudWatch metrics namespace + "resultDestination": "DEDICATED_LOG_GROUP" | "SOURCE_LOG_GROUP" + // DEDICATED_LOG_GROUP writes to a + // dedicated result group, creating it if + // needed; SOURCE_LOG_GROUP writes back to + // the groups the traces were sampled from + } + } + +A role you supply with --role-arn is never edited by the CLI, so it must already +grant logs:PutLogEvents on the destination — plus logs:CreateLogGroup when the +group does not exist yet. + +API reference: + https://docs.aws.amazon.com/bedrock-agentcore-control/latest/APIReference/API_OutputConfig.html + +Example: + --output-config '{"cloudWatchConfig":{"logGroupName":"/company/agent-evaluations","metricsNamespace":"Company/AgentEvaluations","resultDestination":"DEDICATED_LOG_GROUP"}}'`; + +export class OnlineEvalOutputConfigFlag { + static readonly flags = [ + flag( + "output-config", + "where results and metrics are written (JSON OutputConfig)", + z.string().optional(), + { group: "Result output:", help: outputConfigHelp }, + ), + ] as const; + + // Takes a shared SourceResolver so the single-stdin guard spans every + // stdin-capable flag on the command, not just this one. + static async resolve( + value: string | undefined, + resolver: SourceResolver, + ): Promise { + return parseJsonFlag( + "output-config", + await resolver.resolveText("output-config", value), + ); + } +} diff --git a/src/handlers/eval/online-eval/update/index.tsx b/src/handlers/eval/online-eval/update/index.tsx index a5e4eefbf..8dabe1ef9 100644 --- a/src/handlers/eval/online-eval/update/index.tsx +++ b/src/handlers/eval/online-eval/update/index.tsx @@ -8,6 +8,7 @@ import type { Core } from "../../../types"; import { assertMutuallyExclusiveFlags, coreOptsFromCtx, parseJsonFlag } from "../../../utils"; import { filtersHelp } from "../filtersHelp"; import { onlineEvalDataSourceConfigHelp } from "../dataSourceConfigHelp"; +import { OnlineEvalOutputConfigFlag } from "../outputConfig"; const SESSION_SOURCE = "Session source:"; const SOURCE_FILTERS = "Source filters:"; @@ -22,6 +23,12 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => flag("id", "the ID of the online evaluation config to update", z.string().optional(), { group: "Target:", }), + flag( + "description", + "replace the description of the config's monitoring purpose", + z.string().optional(), + { group: "Configuration:" }, + ), flag("agent", "repoint at a different harness ID or Runtime ID", z.string().optional(), { group: SESSION_SOURCE, }), @@ -65,6 +72,7 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => group: EVALUATION, help: filtersHelp, }), + ...OnlineEvalOutputConfigFlag.flags, flag( "role-arn", "replace the IAM role the online evaluation assumes", @@ -89,10 +97,16 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => ); } + // One resolver shared across every stdin-capable flag (--filters, + // --data-source-config, --output-config) so a second `-` is rejected + // rather than reading empty after the first drains stdin. const source = new SourceResolver({ stdin: io.stdin }); + const outputConfig = await OnlineEvalOutputConfigFlag.resolve(flags["output-config"], source); const { response } = await core.eval.updateOnlineEvaluationConfig( flags["id"], { + description: flags["description"], + outputConfig, samplingRate: flags["sampling-rate"], sessionTimeoutMinutes: flags["session-timeout-minutes"], filters: parseJsonFlag( diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index d5a252242..ed52206f6 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -28,6 +28,7 @@ import type { UpdateConfigurationBundleResponse, UpdateEvaluatorResponse, UpdateOnlineEvaluationConfigResponse, + OutputConfig as OnlineEvalOutputConfig, } from "@aws-sdk/client-bedrock-agentcore-control"; import type { CreateABTestResponse, @@ -157,6 +158,8 @@ export type CreateOnlineEvalInput = { evaluatorIds?: string[]; evaluationExecutionRoleArn?: string; enableOnCreate?: boolean; + tags?: Record; + outputConfig?: OnlineEvalOutputConfig; } & ( | { agent: string; endpoint?: string; dataSourceConfig?: undefined } | { agent?: undefined; endpoint?: undefined; dataSourceConfig: DataSourceConfig } @@ -204,6 +207,7 @@ export type DeleteOnlineInsightResponse = DeleteOnlineEvaluationConfigResponse; // `rule` object); `clearEndpoint` nulls out the endpoint scope, falling back to // the agent's default log group. export type UpdateOnlineEvalInput = { + description?: string; samplingRate?: number; sessionTimeoutMinutes?: number; filters?: Rule["filters"]; @@ -219,8 +223,11 @@ export type UpdateOnlineEvalInput = { // Replaces the execution role. Like `harness update`, the CLI never provisions // or re-scopes a role here — a role named here is the caller's to manage. evaluationExecutionRoleArn?: string; + outputConfig?: OnlineEvalOutputConfig; }; +export type { OnlineEvalOutputConfig }; + export type BundleRef = { configBundle: string; bundleVersion: string }; export type CreateConfigBasedABTestInput = {